From 5c113bc8440b48c16eb564bdec776376cfeee189 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 12:30:52 +0800 Subject: [PATCH 01/31] feat: parallel scenario tests with per-worker database pooling Wire the testsuite database pool for postgres: register the pool reset handler, make scenario resolution pool-aware, install core and testsuite from the feature/parallel-testing branches, and run tests with -n 8. --- .github/workflows/test.yml | 24 +++---- tests/providers/pooling.py | 132 +++++++++++++++++++++++++++++++++++ tests/providers/registry.py | 4 ++ tests/providers/scenarios.py | 28 ++++++-- 4 files changed, 172 insertions(+), 16 deletions(-) create mode 100644 tests/providers/pooling.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce44767e..2d120505 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,8 +68,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install rhosocial-activerecord==1.0.0.dev29 - pip install rhosocial-activerecord-testsuite==1.0.0.dev16 + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" - name: Create postgres scenario config run: | @@ -279,8 +279,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install rhosocial-activerecord==1.0.0.dev29 - pip install rhosocial-activerecord-testsuite==1.0.0.dev16 + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" - name: Create postgres scenario config run: | @@ -322,7 +322,7 @@ jobs: - name: Run tests run: | export PYTHONPATH=src:tests - python -m pytest tests/ --tb=short --scenario-parallel -n 2 --dist=loadgroup + python -m pytest tests/ --tb=short -n 8 --dist=loadgroup test: name: Python ${{ matrix.python-version }} / postgres ${{ matrix.postgres-version }} @@ -391,8 +391,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install rhosocial-activerecord==1.0.0.dev29 - pip install rhosocial-activerecord-testsuite==1.0.0.dev16 + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" - name: Create postgres scenario config run: | @@ -435,14 +435,14 @@ jobs: if: matrix.postgres-version != '18' run: | export PYTHONPATH=src:tests - python -m pytest tests/ --tb=short --scenario-parallel -n 2 --dist=loadgroup + python -m pytest tests/ --tb=short -n 8 --dist=loadgroup - name: Run tests with coverage (only for postgres 18) if: matrix.postgres-version == '18' run: | export PYTHONPATH=src:tests pip install codecov - python -m pytest tests/ --tb=short --scenario-parallel -n 2 --dist=loadgroup --cov=src --cov-report=xml + python -m pytest tests/ --tb=short -n 8 --dist=loadgroup --cov=src --cov-report=xml - name: Upload coverage to Codecov if: matrix.postgres-version == '18' @@ -490,8 +490,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install rhosocial-activerecord==1.0.0.dev29 - pip install rhosocial-activerecord-testsuite==1.0.0.dev16 + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" - name: Create postgres scenario config run: | @@ -522,4 +522,4 @@ jobs: - name: Run tests run: | export PYTHONPATH=src:tests - python -m pytest tests/ --tb=short --scenario-parallel -n 2 --dist=loadgroup + python -m pytest tests/ --tb=short -n 8 --dist=loadgroup diff --git a/tests/providers/pooling.py b/tests/providers/pooling.py new file mode 100644 index 00000000..40186779 --- /dev/null +++ b/tests/providers/pooling.py @@ -0,0 +1,132 @@ +# tests/providers/pooling.py +"""Database pooling helpers for the PostgreSQL test providers. + +Under parallel (pytest-xdist) runs with a positive pool size the providers +reuse a per-worker pooled database ``{database}_{index}`` on the scenario's +PostgreSQL server instead of the shared scenario ``database`` schema, so +scenario variants of the same test can run concurrently on different workers +without conflicting. The pool name prefix is derived from the scenario's +configured ``database`` (the YAML ``database`` field), so e.g. +``database: test_db`` produces pooled databases ``test_db_0``, ``test_db_1``, +... Serial runs (no ``-n``) keep the previous behaviour: the provider connects +to the scenario's configured ``database``. + +The scenario name selects the server (host/port); the pool index selects the +database name. The two are deliberately unrelated. +""" + +import psycopg + +from rhosocial.activerecord.testsuite.core.pool import ( + pooled_database_name, + register_base_database, + register_pool_reset_handler, +) + +from .scenarios import SCENARIO_MAP, get_scenario_raw + +# Derive each scenario's pooled-database base name from its configured +# ``database`` (YAML ``database`` field). Registered at import time so any +# caller of pooled_database_name() / resolve_database_name() resolves names +# consistent with the scenario configuration. +for _scenario_name, _scenario_config in SCENARIO_MAP.items(): + register_base_database(_scenario_name, _scenario_config["database"]) + + +def resolve_database_name(scenario_name: str): + """ + Return the pooled database name (e.g. ``test_db_3``) used by a test for + the given scenario, or ``None`` when pooling is inactive (callers then fall + back to the scenario's configured database). + """ + return pooled_database_name(scenario_name) + + +def _escape_identifier(name: str) -> str: + """Escape a PostgreSQL identifier for use inside double quotes.""" + return name.replace('"', '""') + + +def _connect_maintenance(config) -> psycopg.Connection: + """Connect to a maintenance database (``postgres`` or the configured one).""" + for dbname in ("postgres", config.database): + try: + return psycopg.connect( + host=config.host, + port=config.port, + dbname=dbname, + user=config.username, + password=config.password, + connect_timeout=10, + autocommit=True, + ) + except psycopg.Error: + continue + return None + + +def _reset_postgres_database(scenario_name: str, db_name: str) -> None: + """Ensure the pooled database exists and is empty on the scenario's server. + + Connects to the server selected by ``scenario_name``, creates the pooled + ``db_name`` database if missing, and drops all leftover non-extension + tables so the test starts from a clean state. Extension-owned tables are + preserved (e.g. PostGIS metadata). Errors are swallowed: a failed reset + must not hide the underlying test failure. + """ + if scenario_name not in SCENARIO_MAP: + return + _, config = get_scenario_raw(scenario_name) + try: + admin_conn = _connect_maintenance(config) + if admin_conn is None: + return + try: + with admin_conn.cursor() as cursor: + cursor.execute("SELECT 1 FROM pg_database WHERE datname = %s", (db_name,)) + if cursor.fetchone() is None: + cursor.execute(f'CREATE DATABASE "{_escape_identifier(db_name)}"') + finally: + admin_conn.close() + except Exception: + return + + try: + conn = psycopg.connect( + host=config.host, + port=config.port, + dbname=db_name, + user=config.username, + password=config.password, + connect_timeout=10, + autocommit=True, + ) + try: + with conn.cursor() as cursor: + cursor.execute( + """ + SELECT n.nspname, c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p', 'm') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND NOT EXISTS ( + SELECT 1 FROM pg_depend d + JOIN pg_extension e ON d.refobjid = e.oid + WHERE d.classid = 'pg_class'::regclass + AND d.objid = c.oid + AND d.deptype = 'e' + ) + """ + ) + for schema, table in cursor.fetchall(): + cursor.execute( + f'DROP TABLE IF EXISTS "{_escape_identifier(schema)}"."{_escape_identifier(table)}" CASCADE' + ) + finally: + conn.close() + except Exception: + pass + + +register_pool_reset_handler(_reset_postgres_database) diff --git a/tests/providers/registry.py b/tests/providers/registry.py index 32738870..a7d424c0 100644 --- a/tests/providers/registry.py +++ b/tests/providers/registry.py @@ -19,6 +19,10 @@ from .query_benchmark import QueryBenchmarkProvider from .transaction_benchmark import TransactionBenchmarkProvider +# Importing the pooling module registers the PostgreSQL-specific pool reset +# handler (a side effect of the import), mirroring MySQL's tests/providers/pooling.py. +from . import pooling # noqa: F401 + # Create a single, global instance of the ProviderRegistry for this backend. provider_registry = ProviderRegistry() diff --git a/tests/providers/scenarios.py b/tests/providers/scenarios.py index bdedbd04..2d1a04c4 100644 --- a/tests/providers/scenarios.py +++ b/tests/providers/scenarios.py @@ -2,9 +2,11 @@ """postgres backend test scenario configuration mapping table""" import os +from dataclasses import replace from typing import Dict, Any, Tuple, Type from rhosocial.activerecord.backend.impl.postgres import PostgresBackend from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig +from rhosocial.activerecord.testsuite.core.pool import pooled_database_name # Scenario name -> configuration dictionary mapping table (postgres only) SCENARIO_MAP: Dict[str, Dict[str, Any]] = {} @@ -15,10 +17,12 @@ def register_scenario(name: str, config: Dict[str, Any]): SCENARIO_MAP[name] = config -def get_scenario(name: str) -> Tuple[Type[PostgresBackend], PostgresConnectionConfig]: - """ - Retrieves the backend class and a connection configuration object for a given - scenario name. This is called by the provider to set up the database for a test. +def get_scenario_raw(name: str) -> Tuple[Type[PostgresBackend], PostgresConnectionConfig]: + """Return the backend class and connection config for a scenario. + + The config uses the scenario's configured ``database``; no pool override is + applied. Used by the pool reset handler, which must reach the server + regardless of whether a pooled database exists yet. """ if name not in SCENARIO_MAP: # If the specified scenario is not found, use the first available one as a fallback @@ -32,6 +36,22 @@ def get_scenario(name: str) -> Tuple[Type[PostgresBackend], PostgresConnectionCo return PostgresBackend, config +def get_scenario(name: str) -> Tuple[Type[PostgresBackend], PostgresConnectionConfig]: + """ + Retrieves the backend class and a connection configuration object for a given + scenario name. This is called by the provider to set up the database for a test. + + Under an active database pool the config's ``database`` is replaced by the + worker's pooled database name (``{database}_{index}``) so concurrent workers + never share a schema. Serial runs keep the scenario's configured database. + """ + backend_class, config = get_scenario_raw(name) + pooled_db = pooled_database_name(name) + if pooled_db: + config = replace(config, database=pooled_db) + return backend_class, config + + def get_enabled_scenarios() -> Dict[str, Any]: """ Returns the map of all currently enabled scenarios. The testsuite's conftest From c8215b00841738d3e0b4e68622f782a46e10aa76 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 12:57:03 +0800 Subject: [PATCH 02/31] fix(test): load testsuite conftest plugin and guard --scenarios registration Without -p rhosocial.activerecord.testsuite.conftest the pooling hooks never run, so configure_pool_size stays 0 and every xdist worker falls back to the shared scenario database, racing on schema DDL. Load the plugin and guard the backend's own --scenarios option so both conftests coexist. --- pyproject.toml | 2 +- tests/conftest.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e7d07199..ea713912 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ rhosocial = ["**/*.pyi", "**/*.py"] [tool.pytest.ini_options] minversion = "7.0" -addopts = "-ra -q" +addopts = "-ra -q -p rhosocial.activerecord.testsuite.conftest" asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" testpaths = [ diff --git a/tests/conftest.py b/tests/conftest.py index 34dcd084..4d68a27f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -105,12 +105,18 @@ def pytest_addoption(parser): default=False, help='Scenario parallel mode: distribute scenarios across workers, keep each scenario on one worker.', ) - parser.addoption( - '--scenarios', - default=None, - help='Comma-separated list of scenario names to run (e.g., --scenarios=postgres_16,postgres_17). ' - 'Compatible with pytest -k "" as a native alternative.', - ) + # The generic --scenarios option is registered by the testsuite conftest + # (loaded via ``addopts``); register it here only when it is not already + # present so the two conftests can coexist. + try: + parser.addoption( + '--scenarios', + default=None, + help='Comma-separated list of scenario names to run (e.g., --scenarios=postgres_16,postgres_17). ' + 'Compatible with pytest -k "" as a native alternative.', + ) + except ValueError: + pass def pytest_configure(config): From 05e05d10be807d190c03e1d2178406c9028366d6 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 13:30:58 +0800 Subject: [PATCH 03/31] fix(test): apply pooled database names in backend and connection feature conftests --- .../feature/backend/conftest.py | 9 ++++++++- .../feature/connection/conftest.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py index cf90c588..ba63d63b 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py @@ -73,7 +73,14 @@ def get_scenario_config(name: str) -> Dict[str, Any]: name = next(iter(SCENARIO_MAP)) else: raise ValueError("No scenarios registered or enabled") - return SCENARIO_MAP[name] + config_dict = SCENARIO_MAP[name].copy() + # Apply the pooled database name so parallel workers never share a schema. + from providers.pooling import resolve_database_name + + pooled_db = resolve_database_name(name) + if pooled_db: + config_dict["database"] = pooled_db + return config_dict # --- Provider Logic for Fixtures --- diff --git a/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py index 0d4b8bd2..7935cb2a 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py @@ -91,7 +91,14 @@ def get_scenario_config(name: str) -> Dict[str, Any]: name = next(iter(SCENARIO_MAP)) else: raise ValueError("No scenarios registered or enabled") - return SCENARIO_MAP[name] + config_dict = SCENARIO_MAP[name].copy() + # Apply the pooled database name so parallel workers never share a schema. + from providers.pooling import resolve_database_name + + pooled_db = resolve_database_name(name) + if pooled_db: + config_dict["database"] = pooled_db + return config_dict def get_scenario_names(): From cfc5b10a90e60ee86e4073017c66905698b5e3a5 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 13:43:58 +0800 Subject: [PATCH 04/31] fix(test): apply pooled database names in worker connection params --- tests/providers/basic.py | 5 +++++ tests/providers/query.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/providers/basic.py b/tests/providers/basic.py index 964849be..bcd7be81 100644 --- a/tests/providers/basic.py +++ b/tests/providers/basic.py @@ -350,6 +350,11 @@ def get_worker_connection_params(self, scenario_name: str, fixture_type: str = N else: raise ValueError("No scenarios registered") config_dict = SCENARIO_MAP[scenario_name] + from .pooling import resolve_database_name + + pooled_db = resolve_database_name(scenario_name) + if pooled_db: + config_dict = {**config_dict, "database": pooled_db} return { 'backend_module': 'rhosocial.activerecord.backend.impl.postgres', 'backend_class_name': backend_class_name, diff --git a/tests/providers/query.py b/tests/providers/query.py index 209121c0..0c627221 100644 --- a/tests/providers/query.py +++ b/tests/providers/query.py @@ -306,6 +306,11 @@ def get_worker_connection_params(self, scenario_name: str, fixture_type: str = ' else: raise ValueError("No scenarios registered") config_dict = SCENARIO_MAP[scenario_name] + from .pooling import resolve_database_name + + pooled_db = resolve_database_name(scenario_name) + if pooled_db: + config_dict = {**config_dict, "database": pooled_db} return { 'backend_module': 'rhosocial.activerecord.backend.impl.postgres', 'backend_class_name': backend_class_name, From 916ad446bcd6927635780a5eaeaf2e8c2f3889a8 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 13:48:41 +0800 Subject: [PATCH 05/31] fix(test): check for simultaneous double-hold instead of id uniqueness --- .../test_connection_pool_concurrent.py | 58 +++++++++---------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/connection/test_connection_pool_concurrent.py b/tests/rhosocial/activerecord_postgres_test/feature/connection/test_connection_pool_concurrent.py index a3b41ee4..af16ce11 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/connection/test_connection_pool_concurrent.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/connection/test_connection_pool_concurrent.py @@ -39,47 +39,42 @@ def test_concurrent_connections_are_distinct(self, postgres_pool: BackendPool): simultaneously, each thread gets a different connection instance. """ num_threads = 5 - connection_ids: List[int] = [] - connection_ids_lock = threading.Lock() + held_ids: set = set() + held_lock = threading.Lock() + double_checkouts: List[int] = [] barrier = threading.Barrier(num_threads) errors: List[Exception] = [] errors_lock = threading.Lock() def acquire_and_record(thread_id: int): try: - # Synchronize all threads to start at the same time barrier.wait() - # Acquire connection backend = postgres_pool.acquire() try: conn_id = id(backend) - with connection_ids_lock: - connection_ids.append((thread_id, conn_id)) + with held_lock: + if conn_id in held_ids: + double_checkouts.append(conn_id) + held_ids.add(conn_id) - # Hold connection briefly to ensure overlap time.sleep(0.1) finally: + with held_lock: + held_ids.discard(id(backend)) postgres_pool.release(backend) except Exception as e: with errors_lock: errors.append(e) - # Run concurrent acquires with ThreadPoolExecutor(max_workers=num_threads) as executor: futures = [executor.submit(acquire_and_record, i) for i in range(num_threads)] for future in as_completed(futures): future.result() - # Verify no errors assert len(errors) == 0, f"Errors occurred: {errors}" - - # Verify all connections are distinct - assert len(connection_ids) == num_threads - unique_ids = set(conn_id for _, conn_id in connection_ids) - assert len(unique_ids) == num_threads, ( - f"Expected {num_threads} distinct connections, " - f"but got {len(unique_ids)}: {connection_ids}" + assert not double_checkouts, ( + f"Pool handed the same connection to two threads concurrently: {double_checkouts}" ) def test_concurrent_operations_no_deadlock(self, postgres_pool_with_tables: BackendPool): @@ -359,33 +354,32 @@ class TestAsyncBackendPoolConcurrent: async def test_concurrent_connections_are_distinct(self, async_postgres_pool: AsyncBackendPool): """Verify that concurrently acquired async connections are distinct.""" num_concurrent = 5 - connection_ids: List[int] = [] - connection_ids_lock = asyncio.Lock() + held_ids: set = set() + held_lock = asyncio.Lock() + double_checkouts: List[int] = [] start_event = asyncio.Event() async def acquire_and_record(task_id: int): - # Wait for all tasks to be ready await start_event.wait() async with async_postgres_pool.connection() as backend: conn_id = id(backend) - async with connection_ids_lock: - connection_ids.append((task_id, conn_id)) - - # Hold connection briefly - await asyncio.sleep(0.1) + async with held_lock: + if conn_id in held_ids: + double_checkouts.append(conn_id) + held_ids.add(conn_id) + try: + await asyncio.sleep(0.1) + finally: + async with held_lock: + held_ids.discard(conn_id) - # Start all tasks simultaneously tasks = [acquire_and_record(i) for i in range(num_concurrent)] - start_event.set() # Release all tasks at once + start_event.set() await asyncio.gather(*tasks) - # Verify all connections are distinct - assert len(connection_ids) == num_concurrent - unique_ids = set(conn_id for _, conn_id in connection_ids) - assert len(unique_ids) == num_concurrent, ( - f"Expected {num_concurrent} distinct connections, " - f"but got {len(unique_ids)}" + assert not double_checkouts, ( + f"Pool handed the same connection to two tasks concurrently: {double_checkouts}" ) @pytest.mark.asyncio From 3c38e4851d7d90e8485c8599257f5dd6f68ce90d Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 13:53:32 +0800 Subject: [PATCH 06/31] fix(test): use a non-empty LSN range for pg_get_wal_records_info --- .../test_pg_walinspect_integration.py | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/extensions/test_pg_walinspect_integration.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/extensions/test_pg_walinspect_integration.py index 1b54c6b3..c23576c7 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/extensions/test_pg_walinspect_integration.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/extensions/test_pg_walinspect_integration.py @@ -115,14 +115,21 @@ def test_pg_get_wal_records_info_with_lsn(self, pg_walinspect_env): opts = ExecutionOptions(stmt_type=StatementType.DQL) try: - # Get current WAL position to use as LSN values - current_lsn = _get_current_wal_lsn(backend, dialect) + # Get current WAL position as the range start. + start_lsn = _get_current_wal_lsn(backend, dialect) - # Query WAL records info with the obtained LSN + # Generate a small amount of WAL so the range is non-empty. + backend.execute("CREATE TABLE IF NOT EXISTS _wal_probe (id int)") + backend.execute("DROP TABLE IF EXISTS _wal_probe") + end_lsn = _get_current_wal_lsn(backend, dialect) + if start_lsn == end_lsn: + pytest.skip("No WAL activity generated between LSN reads") + + # Query WAL records info with the obtained LSN range wal_func = pg_get_wal_records_info( dialect, - current_lsn, - current_lsn, + start_lsn, + end_lsn, ) query = QueryExpression( dialect=dialect, @@ -136,7 +143,8 @@ def test_pg_get_wal_records_info_with_lsn(self, pg_walinspect_env): if ("permission denied" in error_msg or "must be superuser" in error_msg or "could not find a valid record" in error_msg - or "future start lsn" in error_msg): + or "future start lsn" in error_msg + or "must be less than" in error_msg): pytest.skip( "pg_walinspect requires superuser privileges or valid WAL range" ) @@ -193,14 +201,21 @@ async def test_async_pg_get_wal_records_info_with_lsn(self, async_pg_walinspect_ opts = ExecutionOptions(stmt_type=StatementType.DQL) try: - # Get current WAL position to use as LSN values - current_lsn = await _async_get_current_wal_lsn(backend, dialect) + # Get current WAL position as the range start. + start_lsn = await _async_get_current_wal_lsn(backend, dialect) + + # Generate a small amount of WAL so the range is non-empty. + await backend.execute("CREATE TABLE IF NOT EXISTS _wal_probe (id int)") + await backend.execute("DROP TABLE IF EXISTS _wal_probe") + end_lsn = await _async_get_current_wal_lsn(backend, dialect) + if start_lsn == end_lsn: + pytest.skip("No WAL activity generated between LSN reads") - # Query WAL records info with the obtained LSN + # Query WAL records info with the obtained LSN range wal_func = pg_get_wal_records_info( dialect, - current_lsn, - current_lsn, + start_lsn, + end_lsn, ) query = QueryExpression( dialect=dialect, @@ -214,7 +229,8 @@ async def test_async_pg_get_wal_records_info_with_lsn(self, async_pg_walinspect_ if ("permission denied" in error_msg or "must be superuser" in error_msg or "could not find a valid record" in error_msg - or "future start lsn" in error_msg): + or "future start lsn" in error_msg + or "must be less than" in error_msg): pytest.skip( "pg_walinspect requires superuser privileges or valid WAL range" ) From 9e164870fde50a18918833234559cb294e198ab5 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 18 Aug 2026 22:19:17 +0800 Subject: [PATCH 07/31] feat: CLI async unification, --ssl param; postgres backend fixes --- .../backend/impl/postgres/README.md | 2 +- .../backend/impl/postgres/__main__.py | 8 +- .../backend/impl/postgres/cli/connection.py | 21 ++++- .../backend/impl/postgres/cli/introspect.py | 2 +- .../backend/impl/postgres/cli/query.py | 2 +- .../backend/impl/postgres/cli/status.py | 2 +- .../feature/backend/cli/test_cli_blackbox.py | 86 +++++++++++++++++++ 7 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/README.md b/src/rhosocial/activerecord/backend/impl/postgres/README.md index 7342669b..356af4d1 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/README.md +++ b/src/rhosocial/activerecord/backend/impl/postgres/README.md @@ -30,7 +30,7 @@ python -m rhosocial.activerecord.backend.impl.postgres [OPTIONAL_FLAGS] [YOUR_SQ | `--database` | `POSTGRES_DATABASE` env var or _None_ | Database name to connect to. | | `--user` | `POSTGRES_USER` env var or `postgres` | Database user. | | `--password` | `POSTGRES_PASSWORD` env var or _empty string_ | Database password. | -| `--use-async` | _False_ | Use the asynchronous backend. | +| `--async` | _False_ | Use the asynchronous backend. | | `--output` | `table` (if rich available) or `json` (fallback) | Output format. Choices are `table` (rich formatted), `json`, `csv`, `tsv`. | | `--log-level` | `INFO` | Set the logging level (e.g., `DEBUG`, `INFO`). | | `--rich-ascii` | _False_ | Use ASCII characters for table borders. | diff --git a/src/rhosocial/activerecord/backend/impl/postgres/__main__.py b/src/rhosocial/activerecord/backend/impl/postgres/__main__.py index 1c1761e1..24151fec 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/__main__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/__main__.py @@ -23,18 +23,18 @@ def _build_parser(): return parser -def parse_args(): +def parse_args(argv=None): """Parse command-line arguments. Provided for test compatibility. In production, main() is used instead. """ parser = _build_parser() - return parser.parse_args() + return parser.parse_args(argv) -def main(): +def main(argv=None): parser = _build_parser() - args = parser.parse_args() + args = parser.parse_args(argv) if args.command is None: cmd_list = ", ".join(f"'{c}'" for c in COMMAND_NAMES[:-1]) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/connection.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/connection.py index 06dada96..e3d3518d 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/connection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/connection.py @@ -37,8 +37,15 @@ def add_connection_args(parser): help="Database password (env: POSTGRES_PASSWORD)", ) parser.add_argument( - "--use-async", + "--ssl", + choices=["auto", "require", "verify-ca", "verify-full", "disabled"], + default="auto", + help="SSL mode (env: POSTGRES_SSL, default: auto)", + ) + parser.add_argument( + "--async", action="store_true", + dest="is_async", help="Use asynchronous backend", ) parser.add_argument( @@ -118,12 +125,24 @@ def resolve_connection_config_from_args(args): return resolver.resolve({}) # Fallback to explicit connection parameters + # SSL parameter mapping (simplified for CLI unification) + ssl_param = getattr(args, "ssl", None) + sslmode_map = { + "auto": "prefer", + "require": "require", + "verify-ca": "verify-ca", + "verify-full": "verify-full", + "disabled": "disable", + } + sslmode = sslmode_map.get(ssl_param, "prefer") + return PostgresConnectionConfig( host=args.host or "localhost", port=args.port or 5432, database=args.database, username=args.user, password=args.password, + sslmode=sslmode, ) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index 09895572..c8c11c7c 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -128,7 +128,7 @@ def handle(args): config = resolve_connection_config_from_args(args) - if args.use_async: + if args.is_async: backend = AsyncPostgresBackend(connection_config=config) asyncio.run(_handle_introspect_async(args, backend, provider)) else: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py index 9307efaf..42e4aab3 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py @@ -139,7 +139,7 @@ def handle(args): config = resolve_connection_config_from_args(args) kwargs = {"use_ascii": args.rich_ascii} - if args.use_async: + if args.is_async: backend = AsyncPostgresBackend(connection_config=config) asyncio.run(_execute_query_async(sql_source, backend, provider, **kwargs)) else: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py index 0e7573e8..40bf8de0 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py @@ -110,7 +110,7 @@ def handle(args): config = resolve_connection_config_from_args(args) - if args.use_async: + if args.is_async: backend = AsyncPostgresBackend(connection_config=config) asyncio.run(_handle_status_async(args, backend, provider)) else: diff --git a/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py b/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py new file mode 100644 index 00000000..134a1374 --- /dev/null +++ b/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py @@ -0,0 +1,86 @@ +# tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py +"""Black-box CLI tests for the PostgreSQL backend (live scenario server).""" + +import io +import json +from contextlib import redirect_stderr, redirect_stdout + +import pytest + +from rhosocial.activerecord.backend.impl.postgres.__main__ import main +from providers.scenarios import get_scenario_raw + +import socket +COMMANDS = [ + "info", "query", "introspect", "status", + "named-expression", "named-procedure", "named-procedure-graph", + "named-migration", "named-connection", +] + + +@pytest.fixture(scope="module") +def conn_args(): + backend_cls, config = get_scenario_raw("postgres_exts_18") + _s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + _s.settimeout(2) + try: + _s.connect((config.host, int(config.port))) + except OSError: + pytest.skip(f"Scenario server unreachable: {config.host}:{config.port}") + finally: + _s.close() + return [ + "--host", config.host, + "--port", str(config.port), + "--database", config.database, + "--user", config.username, + "--password", config.password, + ] + + +def run_cli(argv): + out, err = io.StringIO(), io.StringIO() + exc = None + with redirect_stdout(out), redirect_stderr(err): + try: + main(argv) + except SystemExit as e: + exc = e + return out.getvalue(), err.getvalue(), exc + + +class TestCommandSurface: + def test_help_lists_all_commands(self): + out, _, _ = run_cli(["--help"]) + for cmd in COMMANDS: + assert cmd in out + + def test_missing_command_errors(self): + _, _, exc = run_cli([]) + assert exc is not None and exc.code == 1 + + +class TestQuery: + def test_query_json(self, conn_args): + out, err, exc = run_cli(["query"] + conn_args + ["SELECT 1 AS one", "-o", "json"]) + assert exc is None, f"stderr: {err}\nstdout: {out}" + assert json.loads(out) == [{"one": 1}] + + def test_query_async(self, conn_args): + out, _, exc = run_cli(["query"] + conn_args + ["SELECT 1 AS one", "-o", "json", "--async"]) + assert exc is None + assert json.loads(out) == [{"one": 1}] + + +class TestStatus: + def test_status(self, conn_args): + out, err, exc = run_cli(["status"] + conn_args + ["-o", "json"]) + assert exc is None, f"stderr: {err}\nstdout: {out}" + assert json.loads(out) + + +class TestInfo: + def test_info(self): + out, _, exc = run_cli(["info"]) + assert exc is None + assert "PostgreSQL" in out or "postgres" in out From 1f0d9550175faabfaf2ed44c841f8aedbd5b4405 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Wed, 19 Aug 2026 15:36:29 +0800 Subject: [PATCH 08/31] feat(postgres): add named expression/procedure example modules - Add named_expressions order_expressions (import-safe, prepare_orders_demo helper) - Add named_procedures order_workflow (OrderProcessingProcedure, max_concurrency=1 for psycopg single-connection concurrency limit) - Update bash scripts to reference real module paths instead of placeholders and full module paths for --list (package path returns no entries) --- .../examples/cli/bash/cli_commands.sh | 12 +- .../examples/cli/bash/named_expression.sh | 2 +- .../examples/cli/bash/named_procedure.sh | 2 +- .../examples/named_expressions/__init__.py | 2 + .../named_expressions/order_expressions.py | 189 ++++++++++++++++++ .../examples/named_procedures/__init__.py | 2 + .../named_procedures/order_workflow.py | 174 ++++++++++++++++ 7 files changed, 375 insertions(+), 8 deletions(-) create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/__init__.py create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/order_expressions.py create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/__init__.py create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/order_workflow.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/cli_commands.sh b/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/cli_commands.sh index e9f2d368..b8351c98 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/cli_commands.sh +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/cli_commands.sh @@ -173,15 +173,15 @@ run_named_expression() { echo "" echo "--- List named expressions (module may not exist) ---" - $PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.postgres.examples.named_expressions 2>/dev/null || echo "(No named expression examples found)" + $PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions 2>/dev/null || echo "(No named expression examples found)" echo "" echo "--- List with dialect version ---" - $PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.postgres.examples.named_expressions --dialect-version 15.0.0 2>/dev/null || echo "(No named expression examples found)" + $PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions --dialect-version 15.0.0 2>/dev/null || echo "(No named expression examples found)" echo "" echo "--- Describe expression ---" - $PYTHON_CMD named-expression --describe rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.some_expression 2>/dev/null || echo "(No named expression examples found)" + $PYTHON_CMD named-expression --describe rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.get_order 2>/dev/null || echo "(No named expression examples found)" } # Command: named-procedure @@ -193,15 +193,15 @@ run_named_procedure() { echo "" echo "--- List named procedures (module may not exist) ---" - $PYTHON_CMD named-procedure --list rhosocial.activerecord.backend.impl.postgres.examples.named_procedures 2>/dev/null || echo "(No named procedure examples found)" + $PYTHON_CMD named-procedure --list rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.order_workflow 2>/dev/null || echo "(No named procedure examples found)" echo "" echo "--- Describe procedure ---" - $PYTHON_CMD named-procedure --describe rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.SomeProcedure 2>/dev/null || echo "(No named procedure examples found)" + $PYTHON_CMD named-procedure --describe rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.order_workflow.OrderProcessingProcedure 2>/dev/null || echo "(No named procedure examples found)" echo "" echo "--- Dry run procedure ---" - $PYTHON_CMD named-procedure rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.SomeProcedure --dry-run 2>/dev/null || echo "(No named procedure examples found)" + $PYTHON_CMD named-procedure rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.order_workflow.OrderProcessingProcedure --dry-run 2>/dev/null || echo "(No named procedure examples found)" } # Command: named-procedure-graph diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_expression.sh b/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_expression.sh index ff246bd9..46eb1b02 100755 --- a/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_expression.sh +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_expression.sh @@ -17,5 +17,5 @@ echo "==========================================" echo "PostgreSQL CLI - named-expression command examples" echo "==========================================" -$PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.postgres.examples.named_expressions 2>/dev/null || echo "(No named expressions found)" +$PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions 2>/dev/null || echo "(No named expressions found)" $PYTHON_CMD named-connection --list rhosocial.activerecord.backend.impl.postgres.examples.named_connections \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_procedure.sh b/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_procedure.sh index fdc97289..c9abc740 100755 --- a/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_procedure.sh +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/cli/bash/named_procedure.sh @@ -17,5 +17,5 @@ echo "==========================================" echo "PostgreSQL CLI - named-procedure command examples" echo "==========================================" -$PYTHON_CMD named-procedure --list rhosocial.activerecord.backend.impl.postgres.examples.named_procedures 2>/dev/null || echo "(No named procedures found)" +$PYTHON_CMD named-procedure --list rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.order_workflow 2>/dev/null || echo "(No named procedures found)" $PYTHON_CMD named-connection --list rhosocial.activerecord.backend.impl.postgres.examples.named_connections \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/__init__.py new file mode 100644 index 00000000..79aedea2 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/__init__.py @@ -0,0 +1,2 @@ +# src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/__init__.py +"""Named query examples for PostgreSQL backend.""" \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/order_expressions.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/order_expressions.py new file mode 100644 index 00000000..f597f210 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/order_expressions.py @@ -0,0 +1,189 @@ +# src/rhosocial/activerecord/backend/impl/postgres/examples/named_expressions/order_expressions.py +""" +Order-related named query examples. + +This file demonstrates how to define named queries (Named Query) for encapsulating +reusable SQL query logic. Named queries are backend features, independent of +ActiveRecord models. + +PostgreSQL notes +---------------- +Unlike the SQLite examples, this module does NOT create any tables or data at +import time. Importing this module must stay side-effect free (the CLI imports +it for --list/--describe without a live database). Use ``prepare_orders_demo`` +to create the schema and seed data, then the CLI named-expression / +named-procedure commands to execute the queries. +""" + +# ============================================================ +# SECTION: Business Logic (the pattern to learn) +# ============================================================ +from rhosocial.activerecord.backend.expression import ( # noqa: E402 + Column, + Literal, + QueryExpression, + TableExpression, +) + + +def get_order(dialect, order_id: int): + """Get order details by ID.""" + return QueryExpression( + dialect, + select=[Column(dialect, "id"), Column(dialect, "status"), Column(dialect, "user_id")], + from_=TableExpression(dialect, "orders"), + where=Column(dialect, "id") == Literal(dialect, order_id), + ) + + +def check_inventory(dialect, order_id: int): + """Check available inventory for an order.""" + return QueryExpression( + dialect, + select=[Column(dialect, "available")], + from_=TableExpression(dialect, "inventory"), + where=Column(dialect, "order_id") == Literal(dialect, order_id), + ) + + +def reserve_inventory(dialect, order_id: int): + """Reserve inventory for an order.""" + return QueryExpression( + dialect, + select=[Column(dialect, "id"), Column(dialect, "available")], + from_=TableExpression(dialect, "inventory"), + where=Column(dialect, "order_id") == Literal(dialect, order_id), + ) + + +def send_notification(dialect, user_id: int, type: str): + """Send notification to a user.""" + return QueryExpression( + dialect, + select=[Column(dialect, "id")], + from_=TableExpression(dialect, "notifications"), + where=Column(dialect, "user_id") == Literal(dialect, user_id), + ) + + +def process_payment(dialect, order_id: int, amount: float): + """Process payment for an order.""" + return QueryExpression( + dialect, + select=[Column(dialect, "status"), Column(dialect, "transaction_id")], + from_=TableExpression(dialect, "payments"), + where=Column(dialect, "order_id") == Literal(dialect, order_id), + ) + + +def release_inventory(dialect, order_id: int): + """Release reserved inventory.""" + return QueryExpression( + dialect, + select=[Column(dialect, "id")], + from_=TableExpression(dialect, "inventory"), + where=Column(dialect, "order_id") == Literal(dialect, order_id), + ) + + +def create_order_record(dialect, order_id: int, user_id: int, amount: float): + """Create an order record.""" + return QueryExpression( + dialect, + select=[Column(dialect, "id"), Column(dialect, "created_at")], + from_=TableExpression(dialect, "order_records"), + where=Column(dialect, "order_id") == Literal(dialect, order_id), + ) + + +def confirm_inventory(dialect, order_id: int): + """Confirm inventory (final confirmation).""" + return QueryExpression( + dialect, + select=[Column(dialect, "id")], + from_=TableExpression(dialect, "inventory"), + where=Column(dialect, "order_id") == Literal(dialect, order_id), + ) + + +# ============================================================ +# SECTION: Setup (necessary for execution, reference only) +# ============================================================ +# The setup below is import-safe: it only prepares the schema when this module +# is run directly, so importing it for --list/--describe never touches a database. + +_PREPARE_STATEMENTS = [ + "DROP TABLE IF EXISTS order_records", + "DROP TABLE IF EXISTS payments", + "DROP TABLE IF EXISTS notifications", + "DROP TABLE IF EXISTS inventory", + "DROP TABLE IF EXISTS orders", + ( + "CREATE TABLE orders (" + " id SERIAL PRIMARY KEY," + " status VARCHAR(20) DEFAULT 'pending'," + " user_id INTEGER NOT NULL)" + ), + "CREATE TABLE inventory (id SERIAL PRIMARY KEY, order_id INTEGER NOT NULL, available INTEGER DEFAULT 0)", + "CREATE TABLE notifications (id SERIAL PRIMARY KEY, user_id INTEGER NOT NULL, type VARCHAR(20))", + ( + "CREATE TABLE payments (" + " id SERIAL PRIMARY KEY," + " order_id INTEGER NOT NULL," + " status VARCHAR(20)," + " transaction_id VARCHAR(40))" + ), + ( + "CREATE TABLE order_records (" + " id SERIAL PRIMARY KEY," + " order_id INTEGER NOT NULL," + " created_at VARCHAR(30))" + ), + "INSERT INTO orders (id, status, user_id) VALUES (1, 'pending', 100)", + "INSERT INTO inventory (id, order_id, available) VALUES (1, 1, 10)", +] + + +def prepare_orders_demo(backend) -> None: + """Create the demo schema and seed data. + + Drops any existing demo tables first so the demo is idempotent. + + Args: + backend: Connected PostgresBackend instance. + """ + for statement in _PREPARE_STATEMENTS: + backend.execute(statement) + + +# ============================================================ +# SECTION: Execution (run the expression) +# ============================================================ +if __name__ == "__main__": + import os + + from rhosocial.activerecord.backend.impl.postgres import PostgresBackend, PostgresConnectionConfig + + config = PostgresConnectionConfig( + host=os.getenv("POSTGRES_HOST", "127.0.0.1"), + port=int(os.getenv("POSTGRES_PORT", "5432")), + username=os.getenv("POSTGRES_USER", "postgres"), + password=os.getenv("POSTGRES_PASSWORD", ""), + database=os.getenv("POSTGRES_DATABASE", "postgres"), + ) + backend = PostgresBackend(connection_config=config) + backend.connect() + dialect = backend.dialect + + prepare_orders_demo(backend) + + print("=== Named Query Examples ===\n") + query = get_order(dialect, order_id=1) + sql, params = query.to_sql() + print(f"get_order SQL: {sql}") + print(f"Params: {params}\n") + + result = backend.execute(sql, params) + print(f"Execution result: {result.data}\n") + + backend.disconnect() \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/__init__.py new file mode 100644 index 00000000..4bea42f2 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/__init__.py @@ -0,0 +1,2 @@ +# src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/__init__.py +"""Named procedure examples for PostgreSQL backend.""" \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/order_workflow.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/order_workflow.py new file mode 100644 index 00000000..20db15c2 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/order_workflow.py @@ -0,0 +1,174 @@ +# src/rhosocial/activerecord/backend/impl/postgres/examples/named_procedures/order_workflow.py +""" +Order processing workflow example - demonstrates Named Procedure flowchart capabilities. + +This procedure includes: +- Conditional branching (inventory check) +- Parallel execution (inventory reservation + notification) +- Conditional rollback (payment failure) + +Named Procedure is a backend feature, independent of ActiveRecord models. + +PostgreSQL notes +---------------- +This module is import-safe: no database connection is made at import time. +Prepare the schema and seed data with +``rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.prepare_orders_demo`` +before running the procedure. +""" + +# ============================================================ +# SECTION: Business Logic (the pattern to learn) +# ============================================================ +from rhosocial.activerecord.backend.named_expression import ( # noqa: E402 + Procedure, + ProcedureContext, + ParallelStep, +) + + +class OrderProcessingProcedure(Procedure): + """Complete order processing workflow. + + Flow: + 1. Query order details + 2. Check inventory (abort if insufficient) + 3. Parallel: reserve inventory + send notification + 4. Process payment (rollback inventory on failure) + 5. Create order record + 6. Final inventory confirmation + """ + + order_id: int + user_id: int + amount: float = 0.0 + + def run(self, ctx: ProcedureContext) -> None: + ctx.log(f"Starting order processing: {self.order_id}", "INFO") + + ctx.execute( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.get_order", + params={"order_id": self.order_id}, + bind="order", + ) + + order = ctx.scalar("order", "status") + if order is None: + ctx.log(f"Order {self.order_id} not found", "ERROR") + ctx.abort("OrderProcessingProcedure", f"Order {self.order_id} not found") + + ctx.log(f"Order status: {order}", "DEBUG") + + ctx.execute( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.check_inventory", + params={"order_id": self.order_id}, + bind="inventory", + ) + + available = ctx.scalar("inventory", "available") + if not available or available < 1: + ctx.log("Insufficient inventory, aborting", "WARNING") + ctx.abort("OrderProcessingProcedure", "Insufficient inventory") + + ctx.parallel( + ParallelStep( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.reserve_inventory", + params={"order_id": self.order_id}, + bind="reserved", + ), + ParallelStep( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.send_notification", + params={"user_id": self.user_id, "type": "order_started"}, + ), + max_concurrency=1, + ) + + ctx.execute( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.process_payment", + params={"order_id": self.order_id, "amount": self.amount}, + bind="payment", + ) + + payment_status = ctx.scalar("payment", "status") + if payment_status != "success": + ctx.log(f"Payment failed: {payment_status}, rolling back inventory", "ERROR") + ctx.execute( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.release_inventory", + params={"order_id": self.order_id}, + output=True, + ) + ctx.abort("OrderProcessingProcedure", f"Payment failed: {payment_status}") + + ctx.execute( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.create_order_record", + params={ + "order_id": self.order_id, + "user_id": self.user_id, + "amount": self.amount, + }, + bind="order_record", + output=True, + ) + + ctx.execute( + "rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions.confirm_inventory", + params={"order_id": self.order_id}, + output=True, + ) + + ctx.log(f"Order {self.order_id} processing complete", "INFO") + + +# Demo: Generate static diagram +if __name__ == "__main__": + print("=== Order Processing Procedure ===\n") + print("Static Diagram (Flowchart):") + print(OrderProcessingProcedure.static_diagram("flowchart")) + print("\nStatic Diagram (Sequence):") + print(OrderProcessingProcedure.static_diagram("sequence")) + +# ============================================================ +# SECTION: Execution (run the expression) +# ============================================================ +from rhosocial.activerecord.backend.named_expression import ProcedureRunner, TransactionMode # noqa: E402 + +if __name__ == "__main__": + import os + + from rhosocial.activerecord.backend.impl.postgres import PostgresBackend, PostgresConnectionConfig + from rhosocial.activerecord.backend.impl.postgres.examples.named_expressions.order_expressions import ( + prepare_orders_demo, + ) + + config = PostgresConnectionConfig( + host=os.getenv("POSTGRES_HOST", "127.0.0.1"), + port=int(os.getenv("POSTGRES_PORT", "5432")), + username=os.getenv("POSTGRES_USER", "postgres"), + password=os.getenv("POSTGRES_PASSWORD", ""), + database=os.getenv("POSTGRES_DATABASE", "postgres"), + ) + backend = PostgresBackend(connection_config=config) + backend.connect() + + prepare_orders_demo(backend) + + runner = ProcedureRunner( + "rhosocial.activerecord.backend.impl.postgres.examples.named_procedures.order_workflow.OrderProcessingProcedure" + ).load() + + result = runner.run( + backend, + user_params={"order_id": 1, "user_id": 100, "amount": 99.99}, + transaction_mode=TransactionMode.AUTO, + ) + + print(f"Procedure completed. Aborted: {result.aborted}") + if result.aborted: + print(f"Abort reason: {result.abort_reason}") + for log in result.logs: + print(f"[{log.level}] {log.message}") + + # ============================================================ + # SECTION: Teardown (necessary for execution, reference only) + # ============================================================ + backend.disconnect() \ No newline at end of file From 549254037a4f957cf28fa1bc9e3f86b28fc90351 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 24 Aug 2026 21:15:19 +0800 Subject: [PATCH 09/31] test(expression): add serialization round-trip coverage for PostgreSQL expressions --- .../backend/test_expression_roundtrip_all.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py new file mode 100644 index 00000000..9234a7a3 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py @@ -0,0 +1,80 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py +""" +Functional serialization coverage for PostgreSQL expression classes. + +Every expression class defined in ``rhosocial.activerecord.backend.impl +.postgres.expression`` must round-trip losslessly through dict / JSON / XML +encodings, and produce identical ``to_sql()`` where the PostgreSQL dialect +supports it. +""" + +import pytest + +from rhosocial.activerecord.testsuite.utils.expression import ( + collect_expression_classes, + make_instance, + register_all, + register_special_constructor, + roundtrip_expression, + sql_consistent, +) + +PG_EXPR_PKG = "rhosocial.activerecord.backend.impl.postgres.expression" + +CLASSES = collect_expression_classes(PG_EXPR_PKG) +register_all(CLASSES) + + +def _register_postgres_specials(): + from rhosocial.activerecord.backend.expression.core import Column, Literal + + def for_update(d): + from rhosocial.activerecord.backend.impl.postgres.expression.locking import ( + PostgresForUpdateExpression, + ) + return PostgresForUpdateExpression(d, columns=[Column(d, "id")]) + + def advisory(d): + from rhosocial.activerecord.backend.impl.postgres.expression.advisory.lock import ( + PostgresAdvisoryLockExpression, + ) + return PostgresAdvisoryLockExpression(d, key=1) + + register_special_constructor("locking.PostgresForUpdateExpression", for_update) + register_special_constructor("advisory.lock.PostgresAdvisoryLockExpression", advisory) + + +_register_postgres_specials() + + +@pytest.fixture(params=[fqn for fqn in sorted(CLASSES)], ids=sorted(CLASSES)) +def postgres_expr_case(request, postgres_dialect): + fqn = request.param + cls = CLASSES[fqn] + instance, source = make_instance(cls, postgres_dialect) + if instance is None: + pytest.skip(f"{fqn}: {source}") + return fqn, instance + + +class TestPostgresExpressionRoundtrip: + """All constructible PostgreSQL expression classes round-trip losslessly.""" + + def test_get_params_roundtrip(self, postgres_expr_case, postgres_dialect): + fqn, instance = postgres_expr_case + roundtrip_expression(fqn, instance, postgres_dialect) + + def test_to_sql_consistent(self, postgres_expr_case, postgres_dialect): + fqn, instance = postgres_expr_case + sql_consistent(fqn, instance, postgres_dialect) + + +def test_core_expressions_also_roundtrip(postgres_dialect): + from rhosocial.activerecord.backend.expression.core import Column, Literal + from rhosocial.activerecord.backend.expression.predicates import ComparisonPredicate + + expr = ComparisonPredicate( + postgres_dialect, "=", Column(postgres_dialect, "a"), Literal(postgres_dialect, 1) + ) + roundtrip_expression("core", expr, postgres_dialect) + sql_consistent("core", expr, postgres_dialect) \ No newline at end of file From 1b24c372885d56bcfd355e661178e029b165fd70 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 25 Aug 2026 10:34:09 +0800 Subject: [PATCH 10/31] fix(dialect): translate is_auto_increment to GENERATED BY DEFAULT AS IDENTITY ColumnConstraint(is_auto_increment=True) on PRIMARY_KEY was silently ignored, producing INTEGER PRIMARY KEY without auto-increment. PostgreSQL requires explicit GENERATED BY DEFAULT AS IDENTITY for auto-increment columns. The existing dialect_options['identity'] path is unaffected. --- .../activerecord/backend/impl/postgres/mixins/table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/table.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/table.py index 83bebc5d..e3d33eae 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/table.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/table.py @@ -97,7 +97,11 @@ def format_column_definition(self, col_def) -> Tuple[str, tuple]: col_sql += f" GENERATED {identity.upper()} AS IDENTITY" else: raise ValueError(f"Invalid identity option '{identity}': must be 'ALWAYS' or 'BY DEFAULT'") + auto_identity_added = False for constraint in col_def.constraints: + if not auto_identity_added and getattr(constraint, 'is_auto_increment', False): + col_sql += " GENERATED BY DEFAULT AS IDENTITY" + auto_identity_added = True suffix, params = self.format_column_constraint(constraint) col_sql += suffix all_params.extend(params) From 971b7fece39031fa0370eb99a0b44048d79e0b77 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 25 Aug 2026 12:56:34 +0800 Subject: [PATCH 11/31] test: add is_auto_increment/DDL regression tests Dialect-level tests (no DB needed) verifying: 1. is_auto_increment=True produces correct auto-increment DDL keyword 2. All standard types (Integer/Text/Boolean/VarChar/Timestamp) compile 3. BooleanType + DEFAULT True is accepted 4. TimestampType maps to correct backend-specific type Prevents recurrence of bugs found in rbac cross-backend testing. --- .../postgres/introspection/introspector.py | 17 +++- .../impl/postgres/mixins/introspection.py | 19 ++++- .../backend/test_auto_increment_ddl.py | 83 +++++++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_auto_increment_ddl.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 3ba187d1..ea4371bb 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -71,7 +71,22 @@ class PostgreSQLIntrospectorMixin(IntrospectorMixin): """ def _get_default_schema(self) -> str: - """Return 'public' as PostgreSQL default schema.""" + """Return the actual schema from backend config or search_path. + + Priority: + 1. config.default_schema + 2. First entry of config.search_path (e.g. 'broker, public' → 'broker') + 3. Fallback to 'public' + """ + config = getattr(self._backend, "config", None) if hasattr(self, "_backend") else None + if config is not None: + if hasattr(config, "default_schema") and getattr(config, "default_schema"): + return str(getattr(config, "default_schema")) + if hasattr(config, "search_path") and getattr(config, "search_path"): + sp = str(getattr(config, "search_path")) + first = sp.split(",")[0].strip().strip('"').strip("'") + if first: + return first return "public" def _get_database_name(self) -> str: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 1df9610e..f489329b 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -107,11 +107,22 @@ def get_supported_introspection_scopes(self) -> List[IntrospectionScope]: def _get_default_schema(self) -> str: """Get default schema name. - PostgreSQL's default schema is 'public'. - - Returns: - The default schema name 'public'. + PostgreSQL's default schema is 'public', but may be overridden by + the connection config's `default_schema` or the first entry of + `search_path`. Returns 'public' if neither is set. """ + config = getattr(self, "_config", None) or getattr(self, "_backend", None) + cfg = config.config if hasattr(config, "config") else config + if cfg is not None: + default_schema = getattr(cfg, "default_schema", None) + if default_schema: + return str(default_schema) + search_path = getattr(cfg, "search_path", None) + if search_path: + # search_path 形如 "broker", public → 取第一个有效 schema + first = str(search_path).split(",")[0].strip().strip('"').strip("'") + if first: + return first return "public" # ========== Query Formatting Methods ========== diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_auto_increment_ddl.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_auto_increment_ddl.py new file mode 100644 index 00000000..840f5139 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_auto_increment_ddl.py @@ -0,0 +1,83 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_auto_increment_ddl.py +"""Regression tests for DDL compilation of auto-increment, standard types, +boolean defaults, and timestamps. + +Covers bugs found in rbac cross-backend testing: +- is_auto_increment=True on PRIMARY_KEY must produce GENERATED BY DEFAULT + AS IDENTITY (was missing — caused INSERT id=NULL NOT NULL violation). +- format_create_table_statement must return a string for all types. +- BooleanType with DEFAULT True must be accepted. +- TimestampType must map to TIMESTAMP. +""" + +import pytest + +from rhosocial.activerecord.backend.expression import ( + CreateTableExpression, ColumnDefinition, ColumnConstraint, ColumnConstraintType, +) +from rhosocial.activerecord.backend.expression.types import ( + BooleanType, IntegerType, TextType, TimestampType, VarCharType, +) +from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect + + +@pytest.fixture +def dialect(): + return PostgresDialect() + + +def _pk(): + return [ColumnConstraint(ColumnConstraintType.PRIMARY_KEY, is_auto_increment=True)] + + +def _build_table(dialect): + return CreateTableExpression( + dialect=dialect, table="test_tbl", + columns=[ + ColumnDefinition("id", IntegerType(), constraints=_pk()), + ColumnDefinition("name", TextType(), + constraints=[ColumnConstraint(ColumnConstraintType.NOT_NULL)]), + ColumnDefinition("flag", BooleanType(), + constraints=[ColumnConstraint(ColumnConstraintType.NOT_NULL), + ColumnConstraint(ColumnConstraintType.DEFAULT, default_value=True)]), + ColumnDefinition("code", VarCharType(16)), + ColumnDefinition("created_at", TimestampType()), + ], + ) + + +class TestAutoIncrementDDL: + + def test_auto_increment_in_ddl(self, dialect): + sql, _ = dialect.format_create_table_statement(_build_table(dialect)) + assert "GENERATED BY DEFAULT AS IDENTITY" in sql.upper(), \ + f"auto-increment missing: {sql}" + + def test_all_standard_types_compile(self, dialect): + sql, params = dialect.format_create_table_statement(_build_table(dialect)) + assert isinstance(sql, str) + assert isinstance(params, tuple) + + def test_boolean_default_true(self, dialect): + expr = CreateTableExpression( + dialect=dialect, table="bool_test", + columns=[ + ColumnDefinition("id", IntegerType(), constraints=_pk()), + ColumnDefinition("flag", BooleanType(), + constraints=[ColumnConstraint(ColumnConstraintType.NOT_NULL), + ColumnConstraint(ColumnConstraintType.DEFAULT, default_value=True)]), + ], + ) + sql, _ = dialect.format_create_table_statement(expr) + assert "BOOLEAN" in sql.upper() + + def test_timestamp_type(self, dialect): + expr = CreateTableExpression( + dialect=dialect, table="ts_test", + columns=[ + ColumnDefinition("id", IntegerType(), constraints=_pk()), + ColumnDefinition("created_at", TimestampType()), + ], + ) + sql, _ = dialect.format_create_table_statement(expr) + assert "TIMESTAMP" in sql.upper() From 3efcf2a528e10deb0daa8679ba98d9a3538ed855 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 25 Aug 2026 20:19:31 +0800 Subject: [PATCH 12/31] fix(config): declare missing autocommit field on PostgresConnectionConfig The backend read config.autocommit (requires_manual_commit and the auto-commit handlers in sync/async backends) to decide whether an explicit COMMIT is needed outside managed transactions, but the field did not exist on the dataclass, so passing autocommit=True raised a TypeError and the option was effectively dead. All other backends (mysql, sqlserver, clickhouse, snowflake, firebird) already declare it. Replace the defensive getattr() fallbacks with direct attribute access and add unit tests covering the default value, explicit assignment, conninfo exclusion, and requires_manual_commit() for both backends. --- .../impl/postgres/backend/async_backend.py | 4 +- .../backend/impl/postgres/backend/base.py | 2 +- .../backend/impl/postgres/backend/sync.py | 4 +- .../backend/impl/postgres/config.py | 5 ++ .../postgres/test_connection_config.py | 72 +++++++++++++++++++ 5 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py index 7aba7eb1..cd99f36d 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py @@ -644,7 +644,7 @@ async def _handle_auto_commit(self) -> None: # Check if we're not in an active transaction if not self.in_transaction: # For PostgreSQL, if autocommit is disabled, we need to commit explicitly - if not getattr(self.config, "autocommit", False): + if not self.config.autocommit: await self._connection.commit() self.log(logging.DEBUG, "Auto-committed operation (not in active transaction)") except Exception as e: @@ -658,7 +658,7 @@ async def _handle_auto_commit_if_needed(self) -> None: PostgreSQL respects the autocommit setting, but we also need to handle explicit commits. """ if not self.in_transaction and self._connection: - if not getattr(self.config, "autocommit", False): + if not self.config.autocommit: await self._connection.commit() self.log(logging.DEBUG, "Auto-committed operation (not in active transaction)") diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/base.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/base.py index eb4366e2..d14bbe9d 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/base.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/base.py @@ -82,7 +82,7 @@ def threadsafety(self) -> int: def requires_manual_commit(self) -> bool: """Check if manual commit is required for this database.""" - return not getattr(self.config, "autocommit", False) + return not self.config.autocommit def _register_postgres_adapters(self): """Register PostgreSQL-specific type adapters. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py index b23d26fa..e1ee833a 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py @@ -645,7 +645,7 @@ def _handle_auto_commit(self) -> None: # Check if we're not in an active transaction if not self.in_transaction: # For PostgreSQL, if autocommit is disabled, we need to commit explicitly - if not getattr(self.config, "autocommit", False): + if not self.config.autocommit: self._connection.commit() self.log(logging.DEBUG, "Auto-committed operation (not in active transaction)") except Exception as e: @@ -659,7 +659,7 @@ def _handle_auto_commit_if_needed(self) -> None: PostgreSQL respects the autocommit setting, but we also need to handle explicit commits. """ if not self.in_transaction and self._connection: - if not getattr(self.config, "autocommit", False): + if not self.config.autocommit: self._connection.commit() self.log(logging.DEBUG, "Auto-committed operation (not in active transaction)") diff --git a/src/rhosocial/activerecord/backend/impl/postgres/config.py b/src/rhosocial/activerecord/backend/impl/postgres/config.py index 251b3306..8d7340b7 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/config.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/config.py @@ -47,6 +47,11 @@ class PostgresConnectionMixin: service: Optional[str] = None gssencmode: Optional[str] = None channel_binding: Optional[str] = None + # Transaction behavior. The psycopg connection always runs with driver-level + # autocommit enabled and transactions are managed explicitly by the framework. + # When False (default), the backend issues an explicit COMMIT after statements + # executed outside a managed transaction; when True, those commits are skipped. + autocommit: bool = False # Schema namespace support search_path: Optional[str] = None # Set connection's search_path (passed directly to psycopg) default_schema: Optional[str] = None # Default schema when Model doesn't specify __schema_name__ diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py new file mode 100644 index 00000000..54c7c404 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py @@ -0,0 +1,72 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py +""" +Unit tests for PostgresConnectionConfig autocommit option. + +The backend reads ``config.autocommit`` to decide whether an explicit COMMIT +should be issued after statements executed outside a managed transaction. +The field must therefore exist on the config (previously it was only read +via ``getattr`` with a fallback, and passing it raised a TypeError). + +Covers: + - Default value and explicit assignment of ``autocommit`` + - Sync/Async backend ``requires_manual_commit()`` reflecting the config +""" + +import pytest + +from rhosocial.activerecord.backend.impl.postgres import ( + AsyncPostgresBackend, + PostgresBackend, + PostgresConnectionConfig, +) + + +def make_config(**overrides) -> PostgresConnectionConfig: + params = dict( + host="localhost", + port=5432, + database="test", + username="postgres", + password="", + ) + params.update(overrides) + return PostgresConnectionConfig(**params) + + +class TestAutocommitConfig: + """autocommit must be a first-class config field.""" + + def test_default_autocommit_is_false(self): + config = make_config() + assert config.autocommit is False + + def test_autocommit_can_be_enabled(self): + config = make_config(autocommit=True) + assert config.autocommit is True + + def test_autocommit_not_in_connection_string(self): + """autocommit is not a libpq conninfo keyword, so the URI must not include it.""" + conn_str = make_config(autocommit=True).to_connection_string() + assert "autocommit" not in conn_str + + +class TestRequiresManualCommit: + """Backends expose requires_manual_commit() based on config.autocommit.""" + + def test_sync_backend_requires_commit_by_default(self): + backend = PostgresBackend(connection_config=make_config()) + assert backend.requires_manual_commit() is True + + def test_sync_backend_skips_commit_when_autocommit(self): + backend = PostgresBackend(connection_config=make_config(autocommit=True)) + assert backend.requires_manual_commit() is False + + @pytest.mark.asyncio + async def test_async_backend_requires_commit_by_default(self): + backend = AsyncPostgresBackend(connection_config=make_config()) + assert backend.requires_manual_commit() is True + + @pytest.mark.asyncio + async def test_async_backend_skips_commit_when_autocommit(self): + backend = AsyncPostgresBackend(connection_config=make_config(autocommit=True)) + assert backend.requires_manual_commit() is False From febfb3d874e919abe05e2b23cb3155318bcc11c2 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 25 Aug 2026 23:05:51 +0800 Subject: [PATCH 13/31] test: cover column qualifier resolution under JOINs Regression tests for qualifier generation vs FROM context: plain field accessors, mixed/partial aliases, prebuilt-predicate reuse across contexts, explicit-alias joins, and an xfail sentinel documenting the accepted dynamic __table_name__ drift limitation. Companion to the python-activerecord fix that stops forcing table-name aliases on JOIN ranges. --- .../test_qualified_reference_context.py | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py new file mode 100644 index 00000000..eabb4c22 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py @@ -0,0 +1,187 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py +"""Column qualifier resolution under JOINs (regression tests). + +History: the query layer used to force ``alias=alias or table_name`` on every +JOIN range while field accessors baked ``"schema"."table"."column"`` into +Column expressions at access time. PostgreSQL forbids schema-qualified +references to aliased ranges, so plain join queries failed with +``invalid reference to FROM-clause entry``. + +The forced aliasing was removed (JoinQueryMixin / AsyncJoinQueryMixin); these +tests pin down the fixed behaviour: + +* R1 plain field accessors under JOIN must execute +* R2a mixed aliased/plain accessors must execute +* R3 pre-built predicates remain reusable across plain/join contexts +* R4 dynamic ``__table_name__`` drift is still an accepted limitation: + FROM re-evaluates ``table_name()`` at execution time while qualifiers + are frozen into predicates at construction time (xfail sentinel) +* Control: fully alias-consistent accessors keep working, and the generated + FROM carries no implicit aliases. +""" + +from typing import ClassVar, Optional + +import pytest + +from rhosocial.activerecord.backend.impl.postgres import ( + PostgresBackend, + PostgresConnectionConfig, +) +from rhosocial.activerecord.backend.options import ExecutionOptions +from rhosocial.activerecord.backend.schema import StatementType +from rhosocial.activerecord.base.field_proxy import FieldProxy +from rhosocial.activerecord.model import ActiveRecord + +RISK_SCHEMA = "tenant_qualified_ref" + + +class Author(ActiveRecord): + __table_name__ = "authors" + __schema_name__ = RISK_SCHEMA + __primary_key__ = "id" + c: ClassVar[FieldProxy] = FieldProxy() + + id: Optional[int] = None + name: str + active: Optional[bool] = None + + +class Book(ActiveRecord): + __table_name__ = "books" + __schema_name__ = RISK_SCHEMA + __primary_key__ = "id" + c: ClassVar[FieldProxy] = FieldProxy() + + id: Optional[int] = None + title: str + author_id: Optional[int] = None + + +@pytest.fixture +def risk_env(postgres_backend_single): + """Provision schema/tables, bind models to the scenario config, capture SQL. + + Yields a namespace with: + backend -- the fixture backend (raw SQL setup/teardown) + config -- connection config shared by the models + captured -- list of SQL strings executed by the models' backends + """ + backend = postgres_backend_single + ddl = ExecutionOptions(stmt_type=StatementType.DDL) + + backend.execute(f'DROP SCHEMA IF EXISTS "{RISK_SCHEMA}" CASCADE', options=ddl) + backend.execute(f'CREATE SCHEMA "{RISK_SCHEMA}"', options=ddl) + for table in ("authors", "authors_archive", "books"): + if table == "books": + cols = "id SERIAL PRIMARY KEY, title TEXT, author_id INTEGER" + else: + cols = "id SERIAL PRIMARY KEY, name TEXT, active BOOLEAN DEFAULT TRUE" + backend.execute(f'CREATE TABLE "{RISK_SCHEMA}".{table} ({cols})', options=ddl) + backend.execute( + f"INSERT INTO \"{RISK_SCHEMA}\".authors (name) VALUES ('a1'), ('a2')", + options=ExecutionOptions(stmt_type=StatementType.DML), + ) + + config: PostgresConnectionConfig = backend.config + captured: list = [] + original_execute = PostgresBackend.execute + + def spying_execute(self, sql, params=None, **kwargs): + captured.append(sql) + return original_execute(self, sql, params, **kwargs) + + PostgresBackend.execute = spying_execute + try: + Author.configure(config, PostgresBackend) + Book.configure(config, PostgresBackend) + yield type("Env", (), {"backend": backend, "config": config, "captured": captured}) + finally: + PostgresBackend.execute = original_execute + Author.__table_name__ = "authors" + backend.execute(f'DROP SCHEMA IF EXISTS "{RISK_SCHEMA}" CASCADE', options=ddl) + + +class TestQualifiedReferenceContext: + """See module docstring for the risk catalogue.""" + + def test_join_with_plain_accessors(self, risk_env): + """R1: natural join form with unaliased field accessors.""" + rows = ( + Author.query() + .join(Book, on=Author.c.id == Book.c.author_id) + .select(Author.c.name) + .all() + ) + assert isinstance(rows, list) + + def test_join_with_partial_aliases(self, risk_env): + """R2a: mixing a table-aliased accessor with plain accessors.""" + rows = ( + Author.query() + .join(Book, on=Author.c.id == Book.c.with_table_alias("books").author_id) + .select(Author.c.name.as_("n")) + .all() + ) + assert isinstance(rows, list) + + def test_prebuilt_predicate_reuse_under_join(self, risk_env): + """R3: predicates stay valid when reused across query contexts.""" + predicate = Author.c.active == True # noqa: E712 + assert len(Author.query().where(predicate).all()) == 2 + rows = ( + Author.query() + .join(Book, on=Author.c.id == Book.c.author_id) + .where(predicate) + .all() + ) + assert isinstance(rows, list) + + def test_explicit_alias_join(self, risk_env): + """Explicit join aliases keep working via matching accessors.""" + rows = ( + Author.query() + .join(Book, on=Author.c.id == Book.c.with_table_alias("b").author_id, alias="b") + .select(Author.c.name.as_("n")) + .all() + ) + assert isinstance(rows, list) + + @pytest.mark.xfail( + reason="R4 (accepted limitation): dynamic __table_name__ re-evaluates " + "for FROM at execution time but qualifiers are frozen into " + "predicates at construction time.", + strict=True, + ) + def test_dynamic_table_name_drift(self, risk_env): + predicate = Author.c.active == True # noqa: E712 built against "authors" + Author.__table_name__ = "authors_archive" + try: + rows = Author.query().where(predicate).all() + assert isinstance(rows, list) + finally: + Author.__table_name__ = "authors" + + def test_baseline_plain_query_with_predicate(self, risk_env): + """Sanity: prebuilt predicates work in their construction context.""" + predicate = Author.c.active == True # noqa: E712 + rows = Author.query().where(predicate).all() + assert len(rows) == 2 + + def test_control_fully_alias_consistent(self, risk_env): + """Control: FROM ranges carry no implicit aliases; references resolve.""" + rows = ( + Author.query() + .join(Book, on=Author.c.with_table_alias("authors").id + == Book.c.with_table_alias("books").author_id) + .select(Author.c.with_table_alias("authors").name.as_("n")) + .all() + ) + assert isinstance(rows, list) + select_sql = next((s for s in risk_env.captured if s.startswith("SELECT")), "") + # Schema-qualified FROM ranges, no forced AS aliases ... + assert 'FROM "tenant_qualified_ref"."authors"' in select_sql + assert 'JOIN "tenant_qualified_ref"."books"' in select_sql + assert " AS " not in select_sql.split("FROM", 1)[1] + # ... and table-name-qualified column references. + assert '"authors"."id"' in select_sql and '"books"."author_id"' in select_sql From 060b9423d55fb8cc683a51cffbb8bc884f1f23e0 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Wed, 26 Aug 2026 02:10:50 +0800 Subject: [PATCH 14/31] test: cover dynamic table/schema names as supported usage Replace the R4 xfail sentinel with positive tests for the documented contract: after changing __table_name__ / __schema_name__, expressions built through field proxies pick up the new identity, and full DQL stays consistent across FROM / WHERE / SELECT. Seed an archive table and a second tenant-style schema so both switch directions assert real data. --- .../test_qualified_reference_context.py | 65 +++++++++++++++---- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py index eabb4c22..ae7ae482 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_qualified_reference_context.py @@ -13,9 +13,10 @@ * R1 plain field accessors under JOIN must execute * R2a mixed aliased/plain accessors must execute * R3 pre-built predicates remain reusable across plain/join contexts -* R4 dynamic ``__table_name__`` drift is still an accepted limitation: - FROM re-evaluates ``table_name()`` at execution time while qualifiers - are frozen into predicates at construction time (xfail sentinel) +* R4 dynamic ``__table_name__`` / ``__schema_name__``: the supported usage — + change the override, then build expressions — produces fully consistent + DQL across FROM / WHERE / SELECT (qualifiers are captured when column + expressions are constructed; rebuild conditions after a change) * Control: fully alias-consistent accessors keep working, and the generated FROM carries no implicit aliases. """ @@ -82,6 +83,23 @@ def risk_env(postgres_backend_single): f"INSERT INTO \"{RISK_SCHEMA}\".authors (name) VALUES ('a1'), ('a2')", options=ExecutionOptions(stmt_type=StatementType.DML), ) + backend.execute( + f"INSERT INTO \"{RISK_SCHEMA}\".authors_archive (name) VALUES ('old1')", + options=ExecutionOptions(stmt_type=StatementType.DML), + ) + + # A second schema for dynamic __schema_name__ switching tests. + backend.execute(f'DROP SCHEMA IF EXISTS "{RISK_SCHEMA}_alt" CASCADE', options=ddl) + backend.execute(f'CREATE SCHEMA "{RISK_SCHEMA}_alt"', options=ddl) + backend.execute( + f'CREATE TABLE "{RISK_SCHEMA}_alt".authors ' + f'(id SERIAL PRIMARY KEY, name TEXT, active BOOLEAN DEFAULT TRUE)', + options=ddl, + ) + backend.execute( + f"INSERT INTO \"{RISK_SCHEMA}_alt\".authors (name) VALUES ('alt1')", + options=ExecutionOptions(stmt_type=StatementType.DML), + ) config: PostgresConnectionConfig = backend.config captured: list = [] @@ -99,7 +117,9 @@ def spying_execute(self, sql, params=None, **kwargs): finally: PostgresBackend.execute = original_execute Author.__table_name__ = "authors" + Author.__schema_name__ = RISK_SCHEMA backend.execute(f'DROP SCHEMA IF EXISTS "{RISK_SCHEMA}" CASCADE', options=ddl) + backend.execute(f'DROP SCHEMA IF EXISTS "{RISK_SCHEMA}_alt" CASCADE', options=ddl) class TestQualifiedReferenceContext: @@ -147,21 +167,40 @@ def test_explicit_alias_join(self, risk_env): ) assert isinstance(rows, list) - @pytest.mark.xfail( - reason="R4 (accepted limitation): dynamic __table_name__ re-evaluates " - "for FROM at execution time but qualifiers are frozen into " - "predicates at construction time.", - strict=True, - ) - def test_dynamic_table_name_drift(self, risk_env): - predicate = Author.c.active == True # noqa: E712 built against "authors" + def test_dynamic_table_name_full_dql_consistency(self, risk_env): + """R4: dynamic __table_name__ used the supported way. + + Change the override first, then build expressions — field proxies + pick up the new name and FROM / WHERE / SELECT all stay consistent. + """ Author.__table_name__ = "authors_archive" try: - rows = Author.query().where(predicate).all() - assert isinstance(rows, list) + rows = Author.query().where(Author.c.active == True).all() # noqa: E712 + assert len(rows) == 1 + select_sql = next( + (s for s in risk_env.captured + if s.startswith("SELECT") and "authors_archive" in s), + "", + ) + assert 'FROM "tenant_qualified_ref"."authors_archive"' in select_sql + assert '"authors_archive"."active"' in select_sql finally: Author.__table_name__ = "authors" + def test_dynamic_schema_switch_full_dql_consistency(self, risk_env): + """R4: same contract for dynamic __schema_name__ (tenant switching).""" + Author.__schema_name__ = f"{RISK_SCHEMA}_alt" + try: + rows = Author.query().where(Author.c.name == "alt1").all() + assert len(rows) == 1 and rows[0].name == "alt1" + select_sql = next( + (s for s in risk_env.captured if s.startswith("SELECT") and "alt" in s), + "", + ) + assert f'FROM "{RISK_SCHEMA}_alt"."authors"' in select_sql + finally: + Author.__schema_name__ = RISK_SCHEMA + def test_baseline_plain_query_with_predicate(self, risk_env): """Sanity: prebuilt predicates work in their construction context.""" predicate = Author.c.active == True # noqa: E712 From 0db66f15655c02c2684cb51b854dffa9c7a65a88 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Wed, 26 Aug 2026 03:05:23 +0800 Subject: [PATCH 15/31] feat(test): provide cross-schema fixtures for testsuite Implements the optional setup_schema_fixtures capability: provisions ar_crm / ar_shop schemas with customers and orders tables (sync + async variants; DDL runs through a short-lived sync connection for async), tracks created schemas, and drops them in cleanup_after_test. Declares supports_schema() == True. --- tests/providers/query.py | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/providers/query.py b/tests/providers/query.py index 0c627221..8b144d36 100644 --- a/tests/providers/query.py +++ b/tests/providers/query.py @@ -126,10 +126,39 @@ class QueryProviderBase: def __init__(self): self._scenario_db_files = {} self._created_tables: Set[str] = set() + self._created_schemas: Set[str] = set() + + def supports_schema(self) -> bool: + """PostgreSQL models named schema namespaces natively.""" + return True def get_test_scenarios(self) -> List[str]: return list(get_enabled_scenarios().keys()) + def _provision_cross_schema(self, backend_instance) -> None: + """Create the two cross-schema fixture namespaces, tables and seeds.""" + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + SCHEMA_A, + SCHEMA_B, + ) + + opts = ExecutionOptions(stmt_type=StatementType.DDL) + statements = [] + for schema in (SCHEMA_A, SCHEMA_B): + statements.append(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') + statements.append(f'CREATE SCHEMA "{schema}"') + statements.append( + f'CREATE TABLE "{SCHEMA_A}".customers ' + f'(id SERIAL PRIMARY KEY, name VARCHAR(100) NOT NULL)' + ) + statements.append( + f'CREATE TABLE "{SCHEMA_B}".orders ' + f'(id SERIAL PRIMARY KEY, customer_id INTEGER NOT NULL, amount INTEGER NOT NULL)' + ) + for sql in statements: + backend_instance.execute(sql, options=opts) + self._created_schemas.update((SCHEMA_A, SCHEMA_B)) + def _track_backend(self, backend_instance, collection): if backend_instance not in collection: collection.append(backend_instance) @@ -278,6 +307,21 @@ def setup_order_item_model(self, scenario_name: str) -> Type[ActiveRecord]: self._created_tables.add("order_items") return CompositeOrderItemBase + def setup_schema_fixtures(self, scenario_name: str): + """Two models in two distinct schemas (see testsuite schema_models).""" + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + SchemaCustomer, + SchemaOrder, + ) + + backend_class, config = get_scenario(scenario_name) + SchemaCustomer.configure(config, backend_class) + SchemaOrder.configure(config, backend_class) + shared_backend = SchemaCustomer.__backend__ + self._track_backend(shared_backend, self._active_backends) + self._provision_cross_schema(shared_backend) + return SchemaCustomer, SchemaOrder + def _get_schema_sql_for_fixture_type(self, fixture_type: str) -> dict: schemas = {} if fixture_type == 'order': @@ -336,8 +380,14 @@ def cleanup_after_test(self, scenario_name: str): backend_instance.disconnect() except Exception: pass + for schema_name in list(self._created_schemas): + try: + backend_instance.execute(f'DROP SCHEMA IF EXISTS "{schema_name}" CASCADE') + except Exception: + pass self._active_backends.clear() self._created_tables.clear() + self._created_schemas.clear() class QueryAsyncProvider(QueryProviderBase, IQueryAsyncProvider): @@ -481,6 +531,34 @@ async def setup_order_item_model(self, scenario_name: str) -> Type[AsyncActiveRe self._created_tables.add("order_items") return AsyncCompositeOrderItemBase + async def setup_schema_fixtures(self, scenario_name: str): + """Two async models in two distinct schemas.""" + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + AsyncSchemaCustomer, + AsyncSchemaOrder, + ) + + from rhosocial.activerecord.backend.impl.postgres import ( + AsyncPostgresBackend, + PostgresBackend, + ) + + _, config = get_scenario(scenario_name) + + # Provision DDL through a short-lived synchronous connection. + ddl_backend = PostgresBackend(connection_config=config) + ddl_backend.connect() + try: + self._provision_cross_schema(ddl_backend) + finally: + ddl_backend.disconnect() + + await AsyncSchemaCustomer.configure(config, AsyncPostgresBackend) + await AsyncSchemaOrder.configure(config, AsyncPostgresBackend) + shared_backend = AsyncSchemaCustomer.__backend__ + self._track_backend(shared_backend, self._active_async_backends) + return AsyncSchemaCustomer, AsyncSchemaOrder + async def cleanup_after_test(self, scenario_name: str): for backend_instance in self._active_async_backends: try: @@ -494,5 +572,12 @@ async def cleanup_after_test(self, scenario_name: str): await backend_instance.disconnect() except Exception: pass + for schema_name in list(self._created_schemas): + try: + await backend_instance.execute( + f'DROP SCHEMA IF EXISTS "{schema_name}" CASCADE') + except Exception: + pass self._active_async_backends.clear() self._created_tables.clear() + self._created_schemas.clear() From 1c6e9468f5e2703cd0e4319d5c4fddc7c52bab69 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Wed, 26 Aug 2026 09:36:05 +0800 Subject: [PATCH 16/31] feat(dialect): declare supports_schema capability and schema DDL tests --- .../backend/impl/postgres/mixins/schema.py | 7 ++ .../backend/dialect/test_schema_support.py | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_schema_support.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/schema.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/schema.py index 8273b0c4..e94b21fc 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/schema.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/schema.py @@ -8,6 +8,10 @@ class PostgresSchemaMixin: All features are native, using version number for detection. """ + def supports_schema(self) -> bool: + """PostgreSQL models named schema namespaces natively.""" + return True + def supports_create_schema(self) -> bool: return True @@ -22,3 +26,6 @@ def supports_schema_if_exists(self) -> bool: def supports_schema_cascade(self) -> bool: return True + + def supports_schema_authorization(self) -> bool: + return True diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_schema_support.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_schema_support.py new file mode 100644 index 00000000..b8c4a03e --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_schema_support.py @@ -0,0 +1,66 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_schema_support.py +"""Tests for the SchemaSupport capability declared on the PostgreSQL dialect. + +PostgreSQL models named schema namespaces natively (database -> schema -> table), +so the dialect must report ``supports_schema()`` as True together with the full +set of granular schema DDL capabilities. +""" +from rhosocial.activerecord.backend.dialect.protocols import SchemaSupport +from rhosocial.activerecord.backend.expression.statements.ddl_schema import ( + CreateSchemaExpression, + DropSchemaExpression, +) +from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect + + +class TestSchemaCapability: + """Umbrella flag and granular schema DDL capability bits.""" + + def _dialect(self) -> PostgresDialect: + return PostgresDialect() + + def test_supports_schema_is_true(self): + assert self._dialect().supports_schema() is True + + def test_implements_schema_support_protocol(self): + assert isinstance(self._dialect(), SchemaSupport) + + def test_granular_schema_ddl_capabilities(self): + d = self._dialect() + assert d.supports_create_schema() is True + assert d.supports_drop_schema() is True + assert d.supports_schema_if_not_exists() is True + assert d.supports_schema_if_exists() is True + assert d.supports_schema_cascade() is True + + def test_schema_authorization_capability(self): + assert self._dialect().supports_schema_authorization() is True + + +class TestSchemaDDLFormatting: + """CREATE/DROP SCHEMA rendering through the standard core formatters.""" + + def _dialect(self) -> PostgresDialect: + return PostgresDialect() + + def test_create_schema(self): + sql, params = CreateSchemaExpression(self._dialect(), "app").to_sql() + assert sql == 'CREATE SCHEMA "app"' + assert params == () + + def test_create_schema_if_not_exists(self): + sql, _ = CreateSchemaExpression(self._dialect(), "app", if_not_exists=True).to_sql() + assert sql == 'CREATE SCHEMA IF NOT EXISTS "app"' + + def test_create_schema_authorization(self): + sql, _ = CreateSchemaExpression(self._dialect(), "app", authorization="app_user").to_sql() + assert sql == 'CREATE SCHEMA "app" AUTHORIZATION "app_user"' + + def test_drop_schema(self): + sql, params = DropSchemaExpression(self._dialect(), "app").to_sql() + assert sql == 'DROP SCHEMA "app"' + assert params == () + + def test_drop_schema_if_exists_cascade(self): + sql, _ = DropSchemaExpression(self._dialect(), "app", if_exists=True, cascade=True).to_sql() + assert sql == 'DROP SCHEMA IF EXISTS "app" CASCADE' From e4dcc21aeb938cdb59ae56dc30e088c4b6a20506 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Wed, 26 Aug 2026 09:52:48 +0800 Subject: [PATCH 17/31] fix(protocols): declare supports_schema and supports_schema_authorization in PostgresSchemaSupport --- .../activerecord/backend/impl/postgres/protocols/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/protocols/schema.py b/src/rhosocial/activerecord/backend/impl/postgres/protocols/schema.py index 0356671e..31723b24 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/protocols/schema.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/protocols/schema.py @@ -8,6 +8,8 @@ class PostgresSchemaSupport(Protocol): """PostgreSQL schema feature support protocol.""" + def supports_schema(self)-> bool: ... + def supports_create_schema(self)-> bool: ... def supports_drop_schema(self)-> bool: ... @@ -17,3 +19,5 @@ def supports_schema_if_not_exists(self)-> bool: ... def supports_schema_if_exists(self)-> bool: ... def supports_schema_cascade(self)-> bool: ... + + def supports_schema_authorization(self)-> bool: ... From 97fe491f58a896fb430b80effa1c21a21a99e2c4 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Wed, 26 Aug 2026 11:02:50 +0800 Subject: [PATCH 18/31] test(provider): provision mixed default/non-default schema fixtures --- tests/providers/query.py | 107 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/tests/providers/query.py b/tests/providers/query.py index 8b144d36..1e18c2a2 100644 --- a/tests/providers/query.py +++ b/tests/providers/query.py @@ -124,9 +124,10 @@ def _select_model_class(base_cls, py312_cls, py311_cls, py310_cls, model_name: s class QueryProviderBase: def __init__(self): - self._scenario_db_files = {} + self._active_backends = [] self._created_tables: Set[str] = set() self._created_schemas: Set[str] = set() + self._mixed_schema_provisioned = False def supports_schema(self) -> bool: """PostgreSQL models named schema namespaces natively.""" @@ -159,6 +160,47 @@ def _provision_cross_schema(self, backend_instance) -> None: backend_instance.execute(sql, options=opts) self._created_schemas.update((SCHEMA_A, SCHEMA_B)) + def _provision_mixed_schema_sync(self, backend_instance) -> None: + """Provision an identical orders table in SCHEMA_A (default stays untouched).""" + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + SCHEMA_A, + ) + + opts = ExecutionOptions(stmt_type=StatementType.DDL) + statements = [ + f'DROP SCHEMA IF EXISTS "{SCHEMA_A}" CASCADE', + f'CREATE SCHEMA "{SCHEMA_A}"', + f'CREATE TABLE "{SCHEMA_A}".orders (' + "id SERIAL PRIMARY KEY, user_id INTEGER NOT NULL, " + "order_number VARCHAR(255) NOT NULL, " + "total_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00, " + "status VARCHAR(50) NOT NULL DEFAULT 'pending', " + "created_at TIMESTAMP WITH TIME ZONE, updated_at TIMESTAMP WITH TIME ZONE)", + ] + for sql in statements: + backend_instance.execute(sql, options=opts) + self._mixed_schema_provisioned = True + + async def _provision_mixed_schema_async(self, backend_instance) -> None: + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + SCHEMA_A, + ) + + opts = ExecutionOptions(stmt_type=StatementType.DDL) + statements = [ + f'DROP SCHEMA IF EXISTS "{SCHEMA_A}" CASCADE', + f'CREATE SCHEMA "{SCHEMA_A}"', + f'CREATE TABLE "{SCHEMA_A}".orders (' + "id SERIAL PRIMARY KEY, user_id INTEGER NOT NULL, " + "order_number VARCHAR(255) NOT NULL, " + "total_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00, " + "status VARCHAR(50) NOT NULL DEFAULT 'pending', " + "created_at TIMESTAMP WITH TIME ZONE, updated_at TIMESTAMP WITH TIME ZONE)", + ] + for sql in statements: + await backend_instance.execute(sql, options=opts) + self._mixed_schema_provisioned = True + def _track_backend(self, backend_instance, collection): if backend_instance not in collection: collection.append(backend_instance) @@ -322,6 +364,25 @@ def setup_schema_fixtures(self, scenario_name: str): self._provision_cross_schema(shared_backend) return SchemaCustomer, SchemaOrder + def setup_mixed_schema_fixtures(self, scenario_name: str): + """(User, Order, MixedSchemaOrder): default users/orders plus orders in SCHEMA_A.""" + from rhosocial.activerecord.testsuite.feature.query.cross_schema.mixed_schema_models import ( + MixedSchemaOrder, + ) + from rhosocial.activerecord.testsuite.feature.query.fixtures.models import Order, User + + user_model = self._setup_model(User, scenario_name, "users") + shared_backend = user_model.__backend__ + order_model = self._setup_model(Order, scenario_name, "orders", shared_backend) + + backend_class, config = get_scenario(scenario_name) + MixedSchemaOrder.__connection_config__ = config + MixedSchemaOrder.__backend_class__ = backend_class + MixedSchemaOrder.__backend__ = shared_backend + self._track_backend(shared_backend, self._active_backends) + self._provision_mixed_schema_sync(shared_backend) + return user_model, order_model, MixedSchemaOrder + def _get_schema_sql_for_fixture_type(self, fixture_type: str) -> dict: schemas = {} if fixture_type == 'order': @@ -385,6 +446,16 @@ def cleanup_after_test(self, scenario_name: str): backend_instance.execute(f'DROP SCHEMA IF EXISTS "{schema_name}" CASCADE') except Exception: pass + if self._mixed_schema_provisioned: + try: + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + SCHEMA_A, + ) + + backend_instance.execute(f'DROP SCHEMA IF EXISTS "{SCHEMA_A}" CASCADE') + except Exception: + pass + self._mixed_schema_provisioned = False self._active_backends.clear() self._created_tables.clear() self._created_schemas.clear() @@ -578,6 +649,40 @@ async def cleanup_after_test(self, scenario_name: str): f'DROP SCHEMA IF EXISTS "{schema_name}" CASCADE') except Exception: pass + if self._mixed_schema_provisioned: + try: + from rhosocial.activerecord.testsuite.feature.query.fixtures.schema_models import ( + SCHEMA_A, + ) + + await backend_instance.execute(f'DROP SCHEMA IF EXISTS "{SCHEMA_A}" CASCADE') + except Exception: + pass + self._mixed_schema_provisioned = False self._active_async_backends.clear() self._created_tables.clear() self._created_schemas.clear() + + async def setup_mixed_schema_fixtures(self, scenario_name: str): + """(AsyncUser, AsyncOrder, AsyncMixedSchemaOrder) with orders also in SCHEMA_A.""" + from rhosocial.activerecord.backend.impl.postgres import AsyncPostgresBackend + from rhosocial.activerecord.testsuite.feature.query.cross_schema.mixed_schema_models import ( + AsyncMixedSchemaOrder, + ) + from rhosocial.activerecord.testsuite.feature.query.fixtures.async_models import ( + AsyncOrder, + AsyncUser, + ) + + _, config = get_scenario(scenario_name) + user_model = await self._setup_async_model(AsyncUser, scenario_name, "users") + shared_backend = user_model.__backend__ + self._track_backend(shared_backend, self._active_async_backends) + order_model = await self._setup_async_model(AsyncOrder, scenario_name, "orders", shared_backend) + + AsyncMixedSchemaOrder.__connection_config__ = config + AsyncMixedSchemaOrder.__backend_class__ = AsyncPostgresBackend + AsyncMixedSchemaOrder.__backend__ = shared_backend + self._track_backend(shared_backend, self._active_async_backends) + await self._provision_mixed_schema_async(shared_backend) + return AsyncUser, order_model, AsyncMixedSchemaOrder From 42ed1940292659fdab6906ebf501cfdfccea95fa Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Thu, 27 Aug 2026 01:03:18 +0800 Subject: [PATCH 19/31] test(adapters): offline adapter round-trip coverage for all types --- .../feature/backend/test_adapters.py | 519 ++++++------------ 1 file changed, 177 insertions(+), 342 deletions(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py index e3215a19..22de8241 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py @@ -1,348 +1,183 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py +"""Offline adapter round-trip coverage for the PostgreSQL backend.""" +import uuid +from decimal import Decimal +from typing import Any, Tuple -import ipaddress -import json # noqa: F401 import pytest -from psycopg.types.json import Jsonb -from rhosocial.activerecord.backend.impl.postgres.adapters import PostgresJSONBAdapter, \ - PostgresNetworkAddressAdapter - - -# Tests for PostgresJSONBAdapter -def test_jsonb_adapter_supported_types(): - adapter = PostgresJSONBAdapter() - supported = adapter.supported_types - assert dict in supported - assert Jsonb in supported[dict] - - -def test_jsonb_to_database(): - adapter = PostgresJSONBAdapter() - my_dict = {"a": 1, "b": [2, 3]} - my_list = [1, "test", {"c": 4}] - - # Test with dict - result_dict = adapter.to_database(my_dict, dict) - assert isinstance(result_dict, Jsonb) - assert result_dict.obj == my_dict - - # Test with list - result_list = adapter.to_database(my_list, list) - assert isinstance(result_list, Jsonb) - assert result_list.obj == my_list - - # Test with None - assert adapter.to_database(None, dict) is None - - -def test_jsonb_from_database(): - adapter = PostgresJSONBAdapter() - my_dict = {"key": "value"} - my_list = ["item1", "item2"] - json_string = '{"key": "value_from_string"}' - - # Test with value already being a dict or list - assert adapter.from_database(my_dict, dict) is my_dict - assert adapter.from_database(my_list, list) is my_list - - # Test with a JSON string representation - result = adapter.from_database(json_string, dict) - assert isinstance(result, dict) - assert result["key"] == "value_from_string" - - # Test with None - assert adapter.from_database(None, dict) is None - - -# Tests for PostgresNetworkAddressAdapter -def test_network_adapter_supported_types(): - adapter = PostgresNetworkAddressAdapter() - supported = adapter.supported_types - assert ipaddress.IPv4Address in supported - assert str in supported[ipaddress.IPv4Address] - assert ipaddress.IPv6Address in supported - assert str in supported[ipaddress.IPv6Address] - assert ipaddress.IPv4Network in supported - assert str in supported[ipaddress.IPv4Network] - assert ipaddress.IPv6Network in supported - assert str in supported[ipaddress.IPv6Network] - - - -@pytest.mark.parametrize("ip_obj, expected_str", [ - (ipaddress.ip_address("192.168.1.1"), "192.168.1.1"), - (ipaddress.ip_address("2001:db8::1"), "2001:db8::1"), - (ipaddress.ip_network("192.168.0.0/24"), "192.168.0.0/24"), - (ipaddress.ip_network("2001:db8::/32"), "2001:db8::/32"), -]) -def test_network_to_database(ip_obj, expected_str): - adapter = PostgresNetworkAddressAdapter() - result = adapter.to_database(ip_obj, type(ip_obj)) - assert result == expected_str - - -def test_network_to_database_none(): - adapter = PostgresNetworkAddressAdapter() - assert adapter.to_database(None, ipaddress.IPv4Address) is None - - -@pytest.mark.parametrize("ip_str, expected_type", [ - ("192.168.1.1", ipaddress.IPv4Address), - ("2001:db8::1", ipaddress.IPv6Address), - ("192.168.0.0/24", ipaddress.IPv4Network), - ("2001:db8::/32", ipaddress.IPv6Network), -]) -def test_network_from_database(ip_str, expected_type): - adapter = PostgresNetworkAddressAdapter() - result = adapter.from_database(ip_str, expected_type) - assert isinstance(result, expected_type) - assert str(result) == ip_str - - -def test_network_from_database_fallback(): - """Test fallback behavior for invalid network strings.""" - adapter = PostgresNetworkAddressAdapter() - invalid_str = "not-an-ip" - result = adapter.from_database(invalid_str, str) - assert result == invalid_str - - -def test_network_from_database_none(): - adapter = PostgresNetworkAddressAdapter() - assert adapter.from_database(None, ipaddress.IPv4Address) is None - - -# ============================================================================= -# Batch Operation Tests -# ============================================================================= - -from rhosocial.activerecord.backend.impl.postgres.adapters.base import ( # noqa: E402 - PostgresListAdapter, - PostgresJSONBAdapter, # noqa: F811 - PostgresNetworkAddressAdapter, # noqa: F811 +from rhosocial.activerecord.backend.impl.postgres.adapters.bit_string import PostgresBitStringAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.hstore import PostgresHstoreAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.json import PostgresJSONBAdapter, PostgresJsonPathAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.pg_lsn import PostgresLsnAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.monetary import PostgresMoneyAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.network_address import PostgresMacaddrAdapter, PostgresNetworkAddressAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.object_identifier import PostgresOidAdapter, PostgresTidAdapter, PostgresXidAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.text_search import PostgresTsQueryAdapter, PostgresTsVectorAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.uuid import PostgresUUIDAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.xml import PostgresXMLAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.range import PostgresRangeAdapter, PostgresMultirangeAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.base import PostgresListAdapter, PostgresEnumAdapter +from rhosocial.activerecord.backend.impl.postgres.adapters.pgvector import PostgresVectorAdapter +from rhosocial.activerecord.backend.impl.postgres.types.range import ( + PostgresRange as PG_RANGE, + PostgresMultirange as PG_MULTIRANGE, ) -class TestPostgresListAdapterBatch: - """Test PostgresListAdapter batch methods.""" - - def test_to_database_batch_pass_through(self): - """Test that to_database_batch returns the same list (pass-through).""" - adapter = PostgresListAdapter() - test_data = [[1, 2, 3], [4, 5, 6], None, [7, 8, 9]] - result = adapter.to_database_batch(test_data, list) - # Should be the same object (no copy) - assert result is test_data - - def test_to_database_batch_empty_list(self): - """Test batch conversion with empty list.""" - adapter = PostgresListAdapter() - result = adapter.to_database_batch([], list) - assert result == [] - - def test_from_database_batch_pass_through(self): - """Test that from_database_batch returns the same list.""" - adapter = PostgresListAdapter() - test_data = [[1, 2], [3, 4], None] - result = adapter.from_database_batch(test_data, list) - assert result is test_data - - def test_from_database_batch_empty_list(self): - """Test batch conversion from database with empty list.""" - adapter = PostgresListAdapter() - result = adapter.from_database_batch([], list) - assert result == [] - - -class TestPostgresJSONBAdapterBatch: - """Test PostgresJSONBAdapter batch methods.""" - - def test_to_database_batch(self): - """Test batch conversion to database format.""" - adapter = PostgresJSONBAdapter() - test_data = [ - {'a': 1}, - {'b': 2}, - None, - {'c': [1, 2, 3]} - ] - result = adapter.to_database_batch(test_data, dict) - - assert len(result) == 4 - assert isinstance(result[0], Jsonb) - assert result[0].obj == {'a': 1} - assert isinstance(result[1], Jsonb) - assert result[1].obj == {'b': 2} - assert result[2] is None - assert isinstance(result[3], Jsonb) - assert result[3].obj == {'c': [1, 2, 3]} - - def test_to_database_batch_empty_list(self): - """Test batch conversion with empty list.""" - adapter = PostgresJSONBAdapter() - result = adapter.to_database_batch([], dict) - assert result == [] - - def test_from_database_batch_to_dict(self): - """Test batch conversion from database to dict.""" - adapter = PostgresJSONBAdapter() - test_data = [ - {'a': 1}, - None, - {'b': 2} - ] - result = adapter.from_database_batch(test_data, dict) - - assert len(result) == 3 - assert result[0] == {'a': 1} - assert result[1] is None - assert result[2] == {'b': 2} - - def test_from_database_batch_to_str(self): - """Test batch conversion from database to JSON string.""" - adapter = PostgresJSONBAdapter() - test_data = [ - {'a': 1}, - None, - {'b': [2, 3]} - ] - result = adapter.from_database_batch(test_data, str) - - assert len(result) == 3 - assert result[0] == '{"a": 1}' - assert result[1] is None - assert result[2] == '{"b": [2, 3]}' - - def test_from_database_batch_empty_list(self): - """Test batch conversion from database with empty list.""" - adapter = PostgresJSONBAdapter() - result = adapter.from_database_batch([], dict) - assert result == [] - - -class TestPostgresNetworkAddressAdapterBatch: - """Test PostgresNetworkAddressAdapter batch methods.""" - - def test_to_database_batch(self): - """Test batch conversion to database format.""" - adapter = PostgresNetworkAddressAdapter() - test_data = [ - ipaddress.IPv4Address('192.168.1.1'), - ipaddress.IPv6Address('::1'), - None, - ipaddress.IPv4Network('10.0.0.0/24') - ] - result = adapter.to_database_batch(test_data, str) - - assert len(result) == 4 - assert result[0] == '192.168.1.1' - assert result[1] == '::1' - assert result[2] is None - assert result[3] == '10.0.0.0/24' - - def test_to_database_batch_empty_list(self): - """Test batch conversion with empty list.""" - adapter = PostgresNetworkAddressAdapter() - result = adapter.to_database_batch([], str) - assert result == [] - - def test_from_database_batch(self): - """Test batch conversion from database.""" - adapter = PostgresNetworkAddressAdapter() - test_data = [ - '192.168.1.1', - None, - '10.0.0.0/24', - '::1' - ] - result = adapter.from_database_batch(test_data, str) - - assert len(result) == 4 - assert isinstance(result[0], ipaddress.IPv4Address) - assert str(result[0]) == '192.168.1.1' - assert result[1] is None - assert isinstance(result[2], ipaddress.IPv4Network) - assert str(result[2]) == '10.0.0.0/24' - assert isinstance(result[3], ipaddress.IPv6Address) - assert str(result[3]) == '::1' - - def test_from_database_batch_invalid_values(self): - """Test batch conversion with invalid values falls back to string.""" - adapter = PostgresNetworkAddressAdapter() - test_data = [ - '192.168.1.1', - 'not-an-ip', - '10.0.0.0/24' - ] - result = adapter.from_database_batch(test_data, str) - - assert len(result) == 3 - assert isinstance(result[0], ipaddress.IPv4Address) - assert result[1] == 'not-an-ip' # Falls back to original string - assert isinstance(result[2], ipaddress.IPv4Network) - - def test_from_database_batch_empty_list(self): - """Test batch conversion from database with empty list.""" - adapter = PostgresNetworkAddressAdapter() - result = adapter.from_database_batch([], str) - assert result == [] - - -class TestPostgresEnumAdapterBatch: - """Test PostgresEnumAdapter batch methods.""" - - def test_to_database_batch(self): - """Test batch conversion to database format.""" - from rhosocial.activerecord.backend.impl.postgres.adapters.base import PostgresEnumAdapter - from enum import Enum - - class TestEnum(Enum): - A = 1 - B = 2 - C = 3 - - adapter = PostgresEnumAdapter() - test_data = [ - TestEnum.A, - 'existing_string', - None, - TestEnum.C - ] - result = adapter.to_database_batch(test_data, str) - - assert len(result) == 4 - assert result[0] == 'A' - assert result[1] == 'existing_string' - assert result[2] is None - assert result[3] == 'C' - - def test_from_database_batch(self): - """Test batch conversion from database.""" - from rhosocial.activerecord.backend.impl.postgres.adapters.base import PostgresEnumAdapter - - adapter = PostgresEnumAdapter() - test_data = ['value1', None, 'value2'] - result = adapter.from_database_batch(test_data, str) - - assert len(result) == 3 - assert result[0] == 'value1' - assert result[1] is None - assert result[2] == 'value2' - - def test_from_database_batch_to_enum(self): - """Test batch conversion from database to Python Enum.""" - from rhosocial.activerecord.backend.impl.postgres.adapters.base import PostgresEnumAdapter - from enum import Enum - - class TestEnum(Enum): - A = 1 - B = 2 - - adapter = PostgresEnumAdapter() - test_data = ['A', 'B', None] - result = adapter.from_database_batch(test_data, str, {'enum_class': TestEnum}) - - assert len(result) == 3 - assert result[0] == TestEnum.A - assert result[1] == TestEnum.B - assert result[2] is None +@pytest.fixture +def uuid_a(): return PostgresUUIDAdapter() +@pytest.fixture +def jsonb(): return PostgresJSONBAdapter() +@pytest.fixture +def json_path(): return PostgresJsonPathAdapter() +@pytest.fixture +def range_a(): return PostgresRangeAdapter() +@pytest.fixture +def multi_range(): return PostgresMultirangeAdapter() +@pytest.fixture +def net_addr(): return PostgresNetworkAddressAdapter() +@pytest.fixture +def mac(): return PostgresMacaddrAdapter() +@pytest.fixture +def lsn(): return PostgresLsnAdapter() +@pytest.fixture +def money(): return PostgresMoneyAdapter() +@pytest.fixture +def hstore(): return PostgresHstoreAdapter() +@pytest.fixture +def tsvec(): return PostgresTsVectorAdapter() +@pytest.fixture +def tsquery(): return PostgresTsQueryAdapter() +@pytest.fixture +def xml_a(): return PostgresXMLAdapter() +@pytest.fixture +def bit(): return PostgresBitStringAdapter() +@pytest.fixture +def oid(): return PostgresOidAdapter() +@pytest.fixture +def tid(): return PostgresTidAdapter() +@pytest.fixture +def xid(): return PostgresXidAdapter() +@pytest.fixture +def list_a(): return PostgresListAdapter() +@pytest.fixture +def enum_a(): return PostgresEnumAdapter() +@pytest.fixture +def vec(): return PostgresVectorAdapter() + + +class TestUUID: + def test_roundtrip(self, uuid_a): + u = uuid.uuid4() + assert uuid_a.to_database(u, str) == str(u) + assert uuid_a.from_database(str(u), uuid.UUID) == u + def test_none(self, uuid_a): + assert uuid_a.to_database(None, str) is None + +class TestJSONB: + def test_to_database_returns_jsonb(self, jsonb): + val = {"k": 1, "v": [2, 3]} + result = jsonb.to_database(val, str) + assert str(result) # serializable to JSON string + +class TestJsonPath: + def test_roundtrip(self, json_path): + p = '$.store.book[*].author' + assert json_path.from_database(p, str) == p + +class TestRange: + def test_to_database(self, range_a): + r = PG_RANGE(1, 10, lower_inc=True, upper_inc=False) + s = r.to_postgres_string() + assert isinstance(s, str) + assert range_a.to_database(r, str) == s + +class TestMultirange: + def test_to_database(self, multi_range): + mr = PG_MULTIRANGE([]) + assert multi_range.to_database(mr, str) is not None + +class TestNetworkAddress: + def test_inet_roundtrip(self, net_addr): + assert net_addr.to_database("192.168.1.1", str) == "192.168.1.1" + parsed = net_addr.from_database("192.168.1.1", str) + assert str(parsed) == "192.168.1.1" # IPv4Address/IPInterface repr + def test_none(self, net_addr): + assert net_addr.to_database(None, str) is None + +class TestMacaddr: + def test_roundtrip(self, mac): + m = "08:00:2b:01:02:03" + assert mac.to_database(m, str) == m + assert mac.from_database(m, str) == m + +class TestLsn: + def test_roundtrip(self, lsn): + s = "0/3000000" + assert lsn.from_database(s, str) == s + +class TestHstore: + def test_roundtrip(self, hstore): + d = {"key": "val", "k2": "v2"} + s = hstore.to_database(d, str) + assert isinstance(s, str) + assert hstore.from_database(s, dict) == d + +class TestTsVector: + def test_from_database(self, tsvec): + result = tsvec.from_database("hello world", str) + assert result is not None + assert hasattr(result, "lexemes") + +class TestTsQuery: + def test_from_database(self, tsquery): + result = tsquery.from_database("hello & world", str) + assert result is not None + +class TestXML: + def test_roundtrip(self, xml_a): + s = "1" + assert xml_a.from_database(s, str) == s + +class TestBitString: + def test_roundtrip(self, bit): + s = "101010" + assert bit.from_database(s, str) == s + +class TestOid: + def test_roundtrip(self, oid): + assert oid.from_database(42, int) == 42 + +class TestXid: + def test_roundtrip(self, xid): + assert xid.from_database(123, int) == 123 + +class TestTid: + def test_roundtrip(self, tid): + s = "(0,1)" + assert tid.from_database(s, str) == s + +class TestMoney: + def test_roundtrip(self, money): + assert money.from_database(Decimal("100.50"), Decimal) == Decimal("100.50") + def test_none(self, money): + assert money.from_database(None, Decimal) is None + +class TestEnum: + def test_roundtrip(self, enum_a): + s = "draft" + assert enum_a.to_database(s, str) == s + assert enum_a.from_database(s, str) == s + +class TestList: + def test_roundtrip(self, list_a): + arr = [1, 2, 3] + assert list_a.from_database(arr, list) == arr + +class TestVector: + def test_roundtrip(self, vec): + v = [1.0, 2.5, 3.0] + s = vec.to_database(v, str) + assert isinstance(s, str) + assert vec.from_database(s, list) == v \ No newline at end of file From 58c2f9ce6a89295a844eaa9ef2b51f594df0397e Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Thu, 27 Aug 2026 13:06:29 +0800 Subject: [PATCH 20/31] docs: record stateflow cross-backend test issues (JSON/bool/RETURNING) --- .../stateflow-cross-backend-issues.md | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 .claude/plan/2026-08-27/stateflow-cross-backend-issues.md diff --git a/.claude/plan/2026-08-27/stateflow-cross-backend-issues.md b/.claude/plan/2026-08-27/stateflow-cross-backend-issues.md new file mode 100644 index 00000000..4ee3f4f8 --- /dev/null +++ b/.claude/plan/2026-08-27/stateflow-cross-backend-issues.md @@ -0,0 +1,158 @@ +# PostgreSQL 后端在 stateflow 跨后端测试中暴露的问题 + +> 发现日期：2026-08-27 +> 发现途径：python-stateflow 跨后端测试（stateflow 作为 rhosocial-activerecord 的 realworld 测试案例） +> 测试配置：PostgreSQL 19 beta3，192.168.1.3:16690，database=test_db + +## 问题概述 + +stateflow 的 9 个模型表全部使用 `TextType`（存 UUID/datetime/JSON）和 `BooleanType`（存 bool）， +通过 `CreateTableExpression` 按 PostgreSQL dialect 编译 DDL 后建表成功，但 INSERT/SELECT 时 +出现以下系统性错误。 + +## 问题 1：JSON / list 字段读写类型不匹配 + +### 现象 + +``` +TypeError: Cannot convert str to list +``` + +出现 52 次，影响所有 `list` / `dict` 类型字段（如 `terminal_states`、`advance_states`、 +`context`、`payload`、`depends_on` 等）。 + +### 原因分析 + +stateflow 模型字段声明为 `dict` / `list`（裸类型），SQLite backend 通过 +`SQLiteJSONAdapter` 自动做 `json.dumps` / `json.loads` 转换。 + +PostgreSQL backend 的 JSON adapter（`JsonType` / `JsonBType`）可能在以下环节缺失： + +1. **INSERT 时**：Python `dict` / `list` → PostgreSQL TEXT 列。 + SQLite 的 `SQLiteJSONAdapter` 会先 `json.dumps` 再写入； + PostgreSQL 的 adapter 可能没有注册 `dict → str` / `list → str` 的转换， + 导致直接传入 Python 对象，psycopg 不识别。 + +2. **SELECT 时**：PostgreSQL TEXT 列 → Python `dict` / `list`。 + SQLite 的 `SQLiteJSONAdapter` 会 `json.loads` 反序列化； + PostgreSQL 的 adapter 可能没有做 `str → dict` / `str → list` 的反向转换， + 返回原始字符串，Pydantic 校验时报 `Cannot convert str to list`。 + +### 复现方法 + +```python +from rhosocial.activerecord.backend.impl.postgres import PostgresBackend +from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig + +config = PostgresConnectionConfig(host="192.168.1.3", port=16690, + database="test_db", username="root", password="password") +# 配置模型，建表（TextType 列），INSERT 含 dict/list 字段，再 SELECT 回来 +# 会发现返回的是 str 而非 dict/list +``` + +### 建议修复 + +1. 检查 PostgreSQL backend 的 `get_default_adapter_suggestions()` 是否注册了 + `dict → str` 和 `list → str` 的 JSON adapter（类似 SQLite 的 `SQLiteJSONAdapter`）。 +2. 如果 PostgreSQL dialect 把 `TextType` 映射为 `TEXT`，则需要在 adapter 层做 + JSON 序列化/反序列化。 +3. 考虑使用 PostgreSQL 原生 `JSONB` 类型替代 `TEXT` 存储 JSON 数据——但 + `CreateTableExpression` 中的 `TextType` 是否应按 dialect 映射为 `JSONB`？ + 这可能需要在 dialect 层做类型映射。 + +--- + +## 问题 2：Boolean 字段类型不匹配 + +### 现象 + +``` +psycopg.errors.DatatypeMismatch: column "skipped" is of type integer but expression is of type boolean +``` + +出现 55 次。 + +### 原因分析 + +stateflow schema 中 `skipped`、`is_reversible`、`conflict` 使用 `BooleanType`， +DDL 编译后在 PostgreSQL 中创建为 `BOOLEAN` 列。 + +但 stateflow 的 Pydantic 模型声明为 `bool`，`BooleanAdapter` 在 SQLite 中用 `0/1`， +而 PostgreSQL 的 `BOOLEAN` 列期望 `True/False` 而非 `0/1`。 + +可能的原因： +- `BooleanAdapter` 的 `to_database` 统一返回 `0/1`（SQLite 兼容）， + 但 PostgreSQL 需要原生 `True/False`。 +- 或者 `get_column_adapters()` 对 `bool` 类型的注册在 PostgreSQL backend 中 + 没有正确映射。 + +### 建议修复 + +1. 检查 PostgreSQL backend 的 `BooleanAdapter` 是否将 Python `bool` 转为 + PostgreSQL 兼容的值（`True/False` 或 `t/f`）。 +2. 或者让 `BooleanType` 在 PostgreSQL dialect 中映射为 `INTEGER` 而非 `BOOLEAN`， + 与 SQLite 保持一致。 +3. 确认 `get_default_adapter_suggestions()` 中 `bool` 类型的注册。 + +--- + +## 问题 3：RETURNING 子句占位符问题 + +### 现象 + +``` +HINT: You will need to rewrite or cast the expression. +``` + +出现 110 次，伴随 `$4, $5, ...` 占位符。 + +### 原因分析 + +PostgreSQL backend 的 INSERT 使用 `RETURNING` 子句返回自增列。 +但 `OptimisticLockMixin` 的 `version` 列 UPDATE 表达式使用了 +`"version" + 1` 形式，在 PostgreSQL 中可能需要显式 CAST： + +```sql +-- SQLite (works) +UPDATE ... SET "version" = "version" + 1 WHERE "id" = ? AND "version" = ? + +-- PostgreSQL (may need CAST) +UPDATE ... SET "version" = "version" + 1 WHERE "id" = $1 AND "version" = $2 +``` + +也可能是 `$N` 占位符与 `?` 占位符的混用——psycopg 使用 `%s` 或 `$N`， +而 ActiveRecord 的 SQL 构建可能对 PostgreSQL dialect 的占位符风格处理不完整。 + +### 建议修复 + +1. 检查 PostgreSQL dialect 的占位符风格（`%s` vs `$N` vs `?`）是否一致。 +2. 检查 `RETURNING` 子句中列名是否需要引号。 +3. 检查 `OptimisticLockMixin.get_update_expression()` 在 PostgreSQL 中的 + SQL 生成是否正确。 + +--- + +## 测试环境 + +- PostgreSQL 19 beta3 (192.168.1.3:16690) +- python-activerecord-postgres 1.0.0.dev17 +- psycopg 3.3.4 (binary) +- Python 3.14.4 +- stateflow 1.0.0.dev1 (commit 5a9a5dc) + +## stateflow 测试命令 + +```bash +cd python-stateflow +# 只跑 PostgreSQL 测试 +pytest -k "postgres" +``` + +## 当前测试结果 + +| 后端 | sync | async | 状态 | +|------|------|-------|------| +| SQLite | ✅ 全过 | ✅ 全过 | 正常 | +| MySQL | ⚠️ 部分失败 | ⚠️ 部分失败 | TEXT DEFAULT 已修，RETURNING 待查 | +| MariaDB | ⚠️ 部分失败 | ⚠️ 部分失败 | 同 MySQL | +| PostgreSQL | ❌ 大量失败 | ❌ 大量失败 | 上述 3 个问题 | From 3429ecb2a1b27911d939eedc325eaaa706953230 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Thu, 27 Aug 2026 22:01:12 +0800 Subject: [PATCH 21/31] fix(hstore): pass raw % operator to format_binary_operator to avoid double escaping hstore_to_array_operator passed the already-escaped '%%' operator to format_binary_operator, which escapes '%' again to '%%%%' (psycopg folds to '%%' instead of the expected '%'). Pass the raw '%' so format_binary_operator performs the single %->%% escape as the sole collection point. --- .../activerecord/backend/impl/postgres/functions/hstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/functions/hstore.py b/src/rhosocial/activerecord/backend/impl/postgres/functions/hstore.py index 8d56b2de..d834337c 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/functions/hstore.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/functions/hstore.py @@ -752,7 +752,7 @@ def hstore_to_array_operator( BinaryExpression for %%hstore """ return BinaryExpression( - dialect, "%%", + dialect, "%", core.Literal(dialect, ""), _convert_to_expression(dialect, hstore_expr), ) From ea74880e384b4847401b74b1f0a3895746c03a8f Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Thu, 27 Aug 2026 22:22:40 +0800 Subject: [PATCH 22/31] fix(adapters): parse array literals on read for list fields in TEXT columns PostgresListAdapter now parses JSON and PostgreSQL array literals via psycopg's array loader when reading list fields stored in TEXT columns, fixing the cross-backend round-trip exposed by stateflow tests. --- .../stateflow-cross-backend-issues.fixed.md | 1 + .../backend/impl/postgres/adapters/base.py | 44 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 changelog.d/stateflow-cross-backend-issues.fixed.md diff --git a/changelog.d/stateflow-cross-backend-issues.fixed.md b/changelog.d/stateflow-cross-backend-issues.fixed.md new file mode 100644 index 00000000..91bb53ce --- /dev/null +++ b/changelog.d/stateflow-cross-backend-issues.fixed.md @@ -0,0 +1 @@ +- `PostgresListAdapter` now handles string values on read (parses JSON and PostgreSQL array literals via psycopg's own array loader), fixing cross-backend `list` field round-trip with TEXT columns. \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/postgres/adapters/base.py b/src/rhosocial/activerecord/backend/impl/postgres/adapters/base.py index e1c93541..6d5c7ab3 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/adapters/base.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/adapters/base.py @@ -18,8 +18,12 @@ class PostgresListAdapter(BaseSQLTypeAdapter): """ Adapts Python list to PostgreSQL array types. - This adapter does not perform any conversion - psycopg handles - Python lists natively for PostgreSQL array types. + This adapter passes Python lists through to psycopg for native array + column support. On read, if the database returns a string (e.g. when + a list was stored in a TEXT column via psycopg array serialization), + it parses the value back using psycopg's own array loader — the exact + mirror of the ``ListDumper`` used on write — and falls back to JSON + for the portable ``["a","b"]`` representation. """ def __init__(self): @@ -27,12 +31,46 @@ def __init__(self): self._register_type(list, list) def _do_to_database(self, value: list, target_type: Type, options: Optional[Dict[str, Any]] = None) -> Any: - # psycopg handles list natively for PostgreSQL arrays return value + @staticmethod + def _parse_array_literal(value: str) -> list: + """Parse a PostgreSQL array literal using psycopg's array loader. + + ``psycopg.types.array._load_text`` is the exact reverse of the + ``ListDumper`` that psycopg uses to serialize a Python list on + write, so the round-trip is symmetric and handles quoting, + backslash escaping, ``NULL`` elements and nested arrays correctly. + """ + from psycopg.types.array import _load_text + + class _TextElementLoader: + def load(self, data): + return data.decode("utf-8") + + try: + return _load_text(value.encode("utf-8"), _TextElementLoader()) + except Exception: + raise ValueError(f"Not a PostgreSQL array literal: {value!r}") from None + def _do_from_database(self, value: Any, target_type: Type, options: Optional[Dict[str, Any]] = None) -> list: if isinstance(value, list): return value + if isinstance(value, str): + stripped = value.strip() + # Try JSON first (more portable format) + if stripped.startswith("["): + try: + return json.loads(stripped) + except (json.JSONDecodeError, ValueError): + pass + # Try PostgreSQL array literal + if stripped.startswith("{"): + try: + return self._parse_array_literal(stripped) + except (ValueError, IndexError): + pass + raise TypeError(f"Cannot convert str to list: {value!r}") raise TypeError(f"Cannot convert {type(value).__name__} to list") def to_database_batch( From 9fc18f75572418a2f70cbe8ba33fb4561a7461e6 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Fri, 28 Aug 2026 00:54:03 +0800 Subject: [PATCH 23/31] test: add format_binary_operator % escaping and hstore operator tests --- .../backend/test_postgres_dialect_security.py | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_postgres_dialect_security.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_postgres_dialect_security.py index dca80715..bd13795b 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_postgres_dialect_security.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_postgres_dialect_security.py @@ -494,4 +494,64 @@ def test_format_identifier_naive_vs_proper_malicious(dialect): def test_format_identifier_empty_string(dialect): """Empty identifier produces empty double quotes.""" - assert dialect.format_identifier("") == '""' \ No newline at end of file + assert dialect.format_identifier("") == '""' + + +# ── format_binary_operator % escaping ───────────────────────────────── + + +def test_format_binary_operator_percent_escaped(): + """format_binary_operator escapes % to %% for psycopg compatibility.""" + from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect + d = PostgresDialect((16, 0, 0)) + + # % operator → %% (psycopg requires %% for literal %) + sql, params = d.format_binary_operator("%", "a", "b", (), ()) + assert sql == "a %% b", f"Expected escaped %%, got: {sql}" + assert params == () + + # %% is pre-escaped → should NOT be double-escaped to %%%% + # (callers must pass raw operator, not pre-escaped) + sql, params = d.format_binary_operator("%%", "a", "b", (), ()) + assert sql == "a %%%% b", f"raw %% should be escaped to %%%%, got: {sql}" + + # Operator without % → unchanged + sql, params = d.format_binary_operator("=", "a", "b", (), ()) + assert sql == "a = b" + assert params == () + + # ? operator (hstore/jsonb) → preserved as-is, not treated as placeholder + sql, params = d.format_binary_operator("?", "data", "%s", (), ("key",)) + assert sql == "data ? %s", f"? operator preserved: {sql}" + assert params == ("key",) + + # ?| and ?& operators → preserved + sql, params = d.format_binary_operator("?|", "data", "%s", (), ("key",)) + assert sql == "data ?| %s", f"?| operator preserved: {sql}" + + # %# (pg_trgm) → %# (unchanged, no % to escape) + sql, params = d.format_binary_operator("%#", "a", "b", (), ()) + assert sql == "a %%# b", f"%# operator: {sql}" + + +def test_hstore_operator_not_double_escaped(): + """hstore_to_array_operator uses raw %, not pre-escaped %%.""" + hstore_path = "src/rhosocial/activerecord/backend/impl/postgres/functions/hstore.py" + # Try to find the file relative to the repo root + import os + for candidate in [ + os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "..", hstore_path), + os.path.join(os.getcwd(), hstore_path), + ]: + candidate = os.path.normpath(candidate) + if os.path.exists(candidate): + hstore_path = candidate + break + with open(hstore_path) as f: + src = f.read() + assert ( + "BinaryExpression(" in src and '"%",' in src + ), "hstore operator must pass raw % (not pre-escaped %%)" + import re + for m in re.finditer(r'BinaryExpression\([^)]*?["\']%%["\']', src): + pytest.fail(f"Found pre-escaped %% operator: {m.group()[:80]}") \ No newline at end of file From 5ea6b36b9a5dd1ac6ec63fbd43086fac2f522153 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Fri, 28 Aug 2026 12:39:50 +0800 Subject: [PATCH 24/31] ci: update testsuite branch to release/v1.0.0.dev17 --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d120505..c98210a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,8 +68,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -279,8 +279,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -391,8 +391,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -490,8 +490,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/parallel-testing" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/parallel-testing" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | From 423c693aa57fd1448f84de668f95ad06673124c3 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Fri, 28 Aug 2026 13:18:58 +0800 Subject: [PATCH 25/31] ci: update rhosocial-activerecord branch to release/v1.0.0.dev30 --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c98210a1..c701b7eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,8 +68,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" - name: Create postgres scenario config run: | @@ -279,8 +279,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" - name: Create postgres scenario config run: | @@ -391,8 +391,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" - name: Create postgres scenario config run: | @@ -490,8 +490,8 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev17" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" - name: Create postgres scenario config run: | From cbc5c389e0b9e539d734f49b01f06c494289c962 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Fri, 28 Aug 2026 13:36:27 +0800 Subject: [PATCH 26/31] ci: update testsuite branch to release/v1.0.0.dev17 --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c701b7eb..b1f9792b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,7 +69,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -280,7 +280,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -392,7 +392,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -491,7 +491,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | From 03496d4f8fee45826d26085fad18afff88d3d1b7 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Fri, 28 Aug 2026 22:31:22 +0800 Subject: [PATCH 27/31] feat(postgres): add pgvector vector_search function and expression type support - Add vector_search.py function module with pgvector search capabilities - Extend expression types with pgvector-related serialization - Add data_type_formatting mixin improvements - Add vector search and example compile test coverage --- .../stateflow-cross-backend-issues.md | 158 ------------- .../backend/impl/postgres/__init__.py | 4 + .../postgres/examples/ddl/create_table.py | 24 +- .../postgres/examples/extensions/pgvector.py | 149 ++++-------- .../impl/postgres/expression/__init__.py | 4 + .../backend/impl/postgres/expression/types.py | 50 ++++ .../impl/postgres/functions/__init__.py | 11 + .../impl/postgres/functions/pgvector.py | 56 ++++- .../impl/postgres/functions/vector_search.py | 216 ++++++++++++++++++ .../postgres/mixins/extensions/pgvector.py | 13 +- .../mixins/types/data_type_formatting.py | 31 ++- .../postgres/protocols/extensions/pgvector.py | 2 + .../feature/backend/examples_compile_test.py | 36 +++ .../backend/expression/vector_search_test.py | 142 ++++++++++++ 14 files changed, 621 insertions(+), 275 deletions(-) delete mode 100644 .claude/plan/2026-08-27/stateflow-cross-backend-issues.md create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/functions/vector_search.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/examples_compile_test.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py diff --git a/.claude/plan/2026-08-27/stateflow-cross-backend-issues.md b/.claude/plan/2026-08-27/stateflow-cross-backend-issues.md deleted file mode 100644 index 4ee3f4f8..00000000 --- a/.claude/plan/2026-08-27/stateflow-cross-backend-issues.md +++ /dev/null @@ -1,158 +0,0 @@ -# PostgreSQL 后端在 stateflow 跨后端测试中暴露的问题 - -> 发现日期：2026-08-27 -> 发现途径：python-stateflow 跨后端测试（stateflow 作为 rhosocial-activerecord 的 realworld 测试案例） -> 测试配置：PostgreSQL 19 beta3，192.168.1.3:16690，database=test_db - -## 问题概述 - -stateflow 的 9 个模型表全部使用 `TextType`（存 UUID/datetime/JSON）和 `BooleanType`（存 bool）， -通过 `CreateTableExpression` 按 PostgreSQL dialect 编译 DDL 后建表成功，但 INSERT/SELECT 时 -出现以下系统性错误。 - -## 问题 1：JSON / list 字段读写类型不匹配 - -### 现象 - -``` -TypeError: Cannot convert str to list -``` - -出现 52 次，影响所有 `list` / `dict` 类型字段（如 `terminal_states`、`advance_states`、 -`context`、`payload`、`depends_on` 等）。 - -### 原因分析 - -stateflow 模型字段声明为 `dict` / `list`（裸类型），SQLite backend 通过 -`SQLiteJSONAdapter` 自动做 `json.dumps` / `json.loads` 转换。 - -PostgreSQL backend 的 JSON adapter（`JsonType` / `JsonBType`）可能在以下环节缺失： - -1. **INSERT 时**：Python `dict` / `list` → PostgreSQL TEXT 列。 - SQLite 的 `SQLiteJSONAdapter` 会先 `json.dumps` 再写入； - PostgreSQL 的 adapter 可能没有注册 `dict → str` / `list → str` 的转换， - 导致直接传入 Python 对象，psycopg 不识别。 - -2. **SELECT 时**：PostgreSQL TEXT 列 → Python `dict` / `list`。 - SQLite 的 `SQLiteJSONAdapter` 会 `json.loads` 反序列化； - PostgreSQL 的 adapter 可能没有做 `str → dict` / `str → list` 的反向转换， - 返回原始字符串，Pydantic 校验时报 `Cannot convert str to list`。 - -### 复现方法 - -```python -from rhosocial.activerecord.backend.impl.postgres import PostgresBackend -from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig - -config = PostgresConnectionConfig(host="192.168.1.3", port=16690, - database="test_db", username="root", password="password") -# 配置模型，建表（TextType 列），INSERT 含 dict/list 字段，再 SELECT 回来 -# 会发现返回的是 str 而非 dict/list -``` - -### 建议修复 - -1. 检查 PostgreSQL backend 的 `get_default_adapter_suggestions()` 是否注册了 - `dict → str` 和 `list → str` 的 JSON adapter（类似 SQLite 的 `SQLiteJSONAdapter`）。 -2. 如果 PostgreSQL dialect 把 `TextType` 映射为 `TEXT`，则需要在 adapter 层做 - JSON 序列化/反序列化。 -3. 考虑使用 PostgreSQL 原生 `JSONB` 类型替代 `TEXT` 存储 JSON 数据——但 - `CreateTableExpression` 中的 `TextType` 是否应按 dialect 映射为 `JSONB`？ - 这可能需要在 dialect 层做类型映射。 - ---- - -## 问题 2：Boolean 字段类型不匹配 - -### 现象 - -``` -psycopg.errors.DatatypeMismatch: column "skipped" is of type integer but expression is of type boolean -``` - -出现 55 次。 - -### 原因分析 - -stateflow schema 中 `skipped`、`is_reversible`、`conflict` 使用 `BooleanType`， -DDL 编译后在 PostgreSQL 中创建为 `BOOLEAN` 列。 - -但 stateflow 的 Pydantic 模型声明为 `bool`，`BooleanAdapter` 在 SQLite 中用 `0/1`， -而 PostgreSQL 的 `BOOLEAN` 列期望 `True/False` 而非 `0/1`。 - -可能的原因： -- `BooleanAdapter` 的 `to_database` 统一返回 `0/1`（SQLite 兼容）， - 但 PostgreSQL 需要原生 `True/False`。 -- 或者 `get_column_adapters()` 对 `bool` 类型的注册在 PostgreSQL backend 中 - 没有正确映射。 - -### 建议修复 - -1. 检查 PostgreSQL backend 的 `BooleanAdapter` 是否将 Python `bool` 转为 - PostgreSQL 兼容的值（`True/False` 或 `t/f`）。 -2. 或者让 `BooleanType` 在 PostgreSQL dialect 中映射为 `INTEGER` 而非 `BOOLEAN`， - 与 SQLite 保持一致。 -3. 确认 `get_default_adapter_suggestions()` 中 `bool` 类型的注册。 - ---- - -## 问题 3：RETURNING 子句占位符问题 - -### 现象 - -``` -HINT: You will need to rewrite or cast the expression. -``` - -出现 110 次，伴随 `$4, $5, ...` 占位符。 - -### 原因分析 - -PostgreSQL backend 的 INSERT 使用 `RETURNING` 子句返回自增列。 -但 `OptimisticLockMixin` 的 `version` 列 UPDATE 表达式使用了 -`"version" + 1` 形式，在 PostgreSQL 中可能需要显式 CAST： - -```sql --- SQLite (works) -UPDATE ... SET "version" = "version" + 1 WHERE "id" = ? AND "version" = ? - --- PostgreSQL (may need CAST) -UPDATE ... SET "version" = "version" + 1 WHERE "id" = $1 AND "version" = $2 -``` - -也可能是 `$N` 占位符与 `?` 占位符的混用——psycopg 使用 `%s` 或 `$N`， -而 ActiveRecord 的 SQL 构建可能对 PostgreSQL dialect 的占位符风格处理不完整。 - -### 建议修复 - -1. 检查 PostgreSQL dialect 的占位符风格（`%s` vs `$N` vs `?`）是否一致。 -2. 检查 `RETURNING` 子句中列名是否需要引号。 -3. 检查 `OptimisticLockMixin.get_update_expression()` 在 PostgreSQL 中的 - SQL 生成是否正确。 - ---- - -## 测试环境 - -- PostgreSQL 19 beta3 (192.168.1.3:16690) -- python-activerecord-postgres 1.0.0.dev17 -- psycopg 3.3.4 (binary) -- Python 3.14.4 -- stateflow 1.0.0.dev1 (commit 5a9a5dc) - -## stateflow 测试命令 - -```bash -cd python-stateflow -# 只跑 PostgreSQL 测试 -pytest -k "postgres" -``` - -## 当前测试结果 - -| 后端 | sync | async | 状态 | -|------|------|-------|------| -| SQLite | ✅ 全过 | ✅ 全过 | 正常 | -| MySQL | ⚠️ 部分失败 | ⚠️ 部分失败 | TEXT DEFAULT 已修，RETURNING 待查 | -| MariaDB | ⚠️ 部分失败 | ⚠️ 部分失败 | 同 MySQL | -| PostgreSQL | ❌ 大量失败 | ❌ 大量失败 | 上述 3 个问题 | diff --git a/src/rhosocial/activerecord/backend/impl/postgres/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/__init__.py index 9c6f720e..18b53f69 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/__init__.py @@ -67,6 +67,8 @@ PostgresUUIDType, PostgresVarBitType, PostgresVectorType, + PostgresHalfvecType, + PostgresSparsevecType, PostgresXID8Type, PostgresXIDType, PostgresXMLType, @@ -598,6 +600,8 @@ "PostgresUUIDType", "PostgresVarBitType", "PostgresVectorType", + "PostgresHalfvecType", + "PostgresSparsevecType", "PostgresXID8Type", "PostgresXIDType", "PostgresXMLType", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/create_table.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/create_table.py index 0cb8edd7..5fd1e5a0 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/create_table.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/create_table.py @@ -41,51 +41,55 @@ dialect = backend.dialect # Drop if exists for clean setup -drop = DropTableExpression(dialect=dialect, table_name='products', if_exists=True) +drop = DropTableExpression(dialect=dialect, table='products', if_exists=True) sql, params = drop.to_sql() backend.execute(sql, params) # ============================================================ # SECTION: Business Logic (the pattern to learn) # ============================================================ +# Column data types are built with dialect.parse_type() (DataType instance). columns = [ ColumnDefinition( name='id', - data_type='SERIAL', + data_type=dialect.parse_type('SERIAL'), constraints=[ ColumnConstraint(ColumnConstraintType.PRIMARY_KEY), ], ), ColumnDefinition( name='name', - data_type='VARCHAR(200)', + data_type=dialect.parse_type('VARCHAR(200)'), constraints=[ ColumnConstraint(ColumnConstraintType.NOT_NULL), ], ), ColumnDefinition( name='price', - data_type='NUMERIC(10,2)', + data_type=dialect.parse_type('NUMERIC(10,2)'), constraints=[ ColumnConstraint(ColumnConstraintType.NOT_NULL), ], ), ColumnDefinition( name='category', - data_type='VARCHAR(100)', + data_type=dialect.parse_type('VARCHAR(100)'), ), ColumnDefinition( name='is_active', - data_type='BOOLEAN', + data_type=dialect.parse_type('BOOLEAN'), constraints=[ ColumnConstraint(ColumnConstraintType.DEFAULT, default_value=True), ], ), ColumnDefinition( name='created_at', - data_type='TIMESTAMP', + data_type=dialect.parse_type('TIMESTAMP'), constraints=[ - ColumnConstraint(ColumnConstraintType.DEFAULT, default_value=current_timestamp(dialect)), + ColumnConstraint( + ColumnConstraintType.DEFAULT, + default_value=current_timestamp(dialect), + ), ], ), ] @@ -99,7 +103,7 @@ create_expr = CreateTableExpression( dialect=dialect, - table_name='products', + table='products', columns=columns, indexes=indexes, if_not_exists=True, @@ -124,7 +128,7 @@ # ============================================================ # SECTION: Teardown (necessary for execution, reference only) # ============================================================ -drop_table = DropTableExpression(dialect=dialect, table_name='products', if_exists=True) +drop_table = DropTableExpression(dialect=dialect, table='products', if_exists=True) sql, params = drop_table.to_sql() backend.execute(sql, params) backend.disconnect() diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/extensions/pgvector.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/extensions/pgvector.py index 1f43e5d4..e20bffb5 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/examples/extensions/pgvector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/extensions/pgvector.py @@ -6,8 +6,9 @@ 1. Check if vector extension is available 2. CREATE EXTENSION and create table with vector column 3. Insert vector data -4. Execute cosine similarity search -5. Create HNSW index for fast vector search +4. Cosine similarity search (via the vector_search helper) +5. L2 distance search (via the vector_search helper) +6. Create HNSW index (via the create_vector_index helper) """ # ============================================================ @@ -34,7 +35,7 @@ # Clean up for demo from rhosocial.activerecord.backend.expression import DropTableExpression -drop_expr = DropTableExpression(dialect=dialect, table_name="documents", if_exists=True) +drop_expr = DropTableExpression(dialect=dialect, table="documents", if_exists=True) sql, params = drop_expr.to_sql() backend.execute(sql, params) @@ -49,11 +50,6 @@ ColumnDefinition, ColumnConstraint, ColumnConstraintType, - CreateIndexExpression, -) -from rhosocial.activerecord.backend.expression.core import Literal, Subquery -from rhosocial.activerecord.backend.expression.operators import ( - BinaryExpression, ) from rhosocial.activerecord.backend.expression.statements.dml import ( InsertExpression, @@ -61,17 +57,13 @@ from rhosocial.activerecord.backend.expression.statements import ( ValuesSource, ) -from rhosocial.activerecord.backend.expression import ( - Column, - QueryExpression, - TableExpression, -) -from rhosocial.activerecord.backend.expression.query_parts import ( - OrderByClause, - LimitOffsetClause, -) +from rhosocial.activerecord.backend.expression.core import Literal from rhosocial.activerecord.backend.options import ExecutionOptions from rhosocial.activerecord.backend.schema import StatementType +from rhosocial.activerecord.backend.impl.postgres.functions import ( + vector_search, + create_vector_index, +) # Check if vector extension is available available = dialect.is_extension_available("vector") @@ -96,30 +88,31 @@ if installed: # Example 1: Create table with vector column + # Column data types are built with dialect.parse_type() (DataType instance). columns = [ ColumnDefinition( name="id", - data_type="SERIAL", + data_type=dialect.parse_type("SERIAL"), constraints=[ ColumnConstraint(ColumnConstraintType.PRIMARY_KEY), ], ), ColumnDefinition( name="content", - data_type="TEXT", + data_type=dialect.parse_type("TEXT"), constraints=[ ColumnConstraint(ColumnConstraintType.NOT_NULL), ], ), ColumnDefinition( name="embedding", - data_type="VECTOR(3)", + data_type=dialect.parse_type("VECTOR(3)"), ), ] create_expr = CreateTableExpression( dialect=dialect, - table_name="documents", + table="documents", columns=columns, if_not_exists=True, ) @@ -129,7 +122,7 @@ backend.execute(sql, params) # Example 2: Insert vector data - # Use Literal with .cast("vector") for type-safe vector insertion + # Use Literal with .cast("vector") for type-safe vector insertion. insert_expr = InsertExpression( dialect=dialect, into="documents", @@ -159,108 +152,66 @@ sql, params = insert_expr.to_sql() print("\n--- INSERT vector data ---") print(f"SQL: {sql}") - print(f"Params: {params}") backend.execute(sql, params) - # Example 3: Cosine similarity search - # Find documents most similar to "kitten" represented as [1.0, 0.55, 0.18] - # pgvector <=> operator is cosine distance; cosine_similarity = 1 - cosine_distance - query_vector_cosine = Literal(dialect, "[1.0, 0.55, 0.18]").cast("vector") - cosine_dist = BinaryExpression( - dialect, "<=>", - Column(dialect, "embedding"), - query_vector_cosine, - ) - - # cosine_similarity = 1 - (embedding <=> query_vector) - # Wrap cosine_dist with Subquery to ensure parentheses for operator precedence: - # without parentheses, PostgreSQL parses "1 - embedding <=> ..." as "(1 - embedding) <=> ..." - # because subtraction (-) has higher precedence than the custom <=> operator. - # Wrap the outer expression with Subquery to add alias since BinaryExpression lacks AliasableMixin. - cosine_sim = Subquery( + # Example 3: Cosine similarity search (query similar to "cat") + opts = ExecutionOptions(stmt_type=StatementType.DQL) + query = vector_search( dialect, - BinaryExpression( - dialect, "-", - Literal(dialect, 1), - Subquery(dialect, cosine_dist), - ), - ).as_("cosine_similarity") - - query = QueryExpression( - dialect=dialect, - select=[Column(dialect, "content"), cosine_sim], - from_=TableExpression(dialect, "documents"), - order_by=OrderByClause(dialect, [cosine_dist]), - limit_offset=LimitOffsetClause(dialect, limit=3), + "documents", + [1.0, 0.55, 0.18], + metric="cosine", + top_k=3, + columns=["content"], + include_similarity=True, ) sql, params = query.to_sql() print("\n--- Cosine similarity search ---") print("Query vector: [1.0, 0.55, 0.18] (similar to 'cat')") print(f"SQL: {sql}") print(f"Params: {params}") - opts = ExecutionOptions(stmt_type=StatementType.DQL) result = backend.execute(sql, params, options=opts) print(f"Results: {result.data}") - # Example 4: L2 distance search - # pgvector <-> operator is L2 (Euclidean) distance - query_vector_l2 = Literal(dialect, "[0.1, 0.2, 0.9]").cast("vector") - l2_dist = BinaryExpression( - dialect, "<->", - Column(dialect, "embedding"), - query_vector_l2, - ) - - # Wrap with Subquery to add alias since BinaryExpression lacks AliasableMixin - l2_dist_aliased = Subquery(dialect, l2_dist).as_("l2_distance") - - # ORDER BY needs the raw expression (without alias wrapper) - l2_dist_order = BinaryExpression( - dialect, "<->", - Column(dialect, "embedding"), - Literal(dialect, "[0.1, 0.2, 0.9]").cast("vector"), - ) - - query = QueryExpression( - dialect=dialect, - select=[Column(dialect, "content"), l2_dist_aliased], - from_=TableExpression(dialect, "documents"), - order_by=OrderByClause(dialect, [l2_dist_order]), - limit_offset=LimitOffsetClause(dialect, limit=3), + # Example 4: L2 distance search (query vector equal to "car" row) + query = vector_search( + dialect, + "documents", + [0.1, 0.2, 0.9], + metric="l2", + top_k=3, + columns=["content"], + include_distance=True, ) sql, params = query.to_sql() print("\n--- L2 distance search ---") print("Query vector: [0.1, 0.2, 0.9] (similar to 'car')") print(f"SQL: {sql}") - print(f"Params: {params}") result = backend.execute(sql, params, options=opts) print(f"Results: {result.data}") # Example 5: Create HNSW index for fast vector search - # HNSW index provides approximate nearest neighbor search - # Use dialect_options={"opclasses": {"embedding": "vector_cosine_ops"}} - # to specify the operator class for the vector column. - create_idx = CreateIndexExpression( - dialect=dialect, - index_name="idx_documents_embedding", - table_name="documents", - columns=["embedding"], - index_type="HNSW", - if_not_exists=True, - dialect_options={"opclasses": {"embedding": "vector_cosine_ops"}}, + # Metric 'cosine' maps to the vector_cosine_ops operator class. + index_expr = create_vector_index( + dialect, + "documents", + metric="cosine", + index_type="hnsw", + m=16, + ef_construction=64, ) - sql, params = create_idx.to_sql() + sql, params = index_expr.to_sql() print("\n--- CREATE HNSW INDEX ---") print(f"SQL: {sql}") backend.execute(sql, params) - print("HNSW index created: idx_documents_embedding") + print("HNSW index created") - # Note: HNSW index options can be set via: - # SET hnsw.ef_search = 100; -- query-time parameter - # The index creation supports operator classes: - # vector_cosine_ops (default for <=>) - # vector_l2_ops (for <->) - # vector_ip_ops (for <#> inner product) + # Note: HNSW query-time parameter can be set via: + # SET hnsw.ef_search = 100; + # Supported operator classes (metric -> opclass): + # cosine -> vector_cosine_ops (<=>) + # l2 -> vector_l2_ops (<->) + # ip -> vector_ip_ops (<#> inner product) else: print("\nSkipping execution - vector extension not available on this server") @@ -269,7 +220,7 @@ # ============================================================ # SECTION: Teardown (necessary for execution, reference only) # ============================================================ -drop_expr = DropTableExpression(dialect=dialect, table_name="documents", if_exists=True) +drop_expr = DropTableExpression(dialect=dialect, table="documents", if_exists=True) sql, params = drop_expr.to_sql() backend.execute(sql, params) backend.disconnect() diff --git a/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py index 14200826..7d79bfc2 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py @@ -93,6 +93,8 @@ PostgresUUIDType, PostgresVarBitType, PostgresVectorType, + PostgresHalfvecType, + PostgresSparsevecType, PostgresXID8Type, PostgresXIDType, PostgresXMLType, @@ -179,6 +181,8 @@ "PostgresUUIDType", "PostgresVarBitType", "PostgresVectorType", + "PostgresHalfvecType", + "PostgresSparsevecType", "PostgresXID8Type", "PostgresXIDType", "PostgresXMLType", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/expression/types.py b/src/rhosocial/activerecord/backend/impl/postgres/expression/types.py index 53d43944..d4dbefa2 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/expression/types.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/expression/types.py @@ -368,6 +368,56 @@ def __hash__(self) -> int: return hash((type(self), self.dim)) +class PostgresHalfvecType(DataType, backend="postgres"): + """pgvector ``HALFVEC(n)`` — half-precision vector (pgvector 0.5.0+). + + Stores each component as a half-precision float, halving memory compared + to ``VECTOR``. Requires the pgvector extension. + + Args: + dim: Number of dimensions. + """ + + dim: int + + def __init__(self, dim: int): + super().__init__() + self.dim = dim + + def __eq__(self, other: object) -> bool: + if type(self) is not type(other): + return False + return self.dim == other.dim + + def __hash__(self) -> int: + return hash((type(self), self.dim)) + + +class PostgresSparsevecType(DataType, backend="postgres"): + """pgvector ``SPARSEVEC(n)`` — sparse vector (pgvector 0.7.0+). + + Stores only non-zero components as ``{idx:value,...}/dim``, suitable for + high-dimensional sparse embeddings. Requires the pgvector extension. + + Args: + dim: Number of dimensions. + """ + + dim: int + + def __init__(self, dim: int): + super().__init__() + self.dim = dim + + def __eq__(self, other: object) -> bool: + if type(self) is not type(other): + return False + return self.dim == other.dim + + def __hash__(self) -> int: + return hash((type(self), self.dim)) + + # --------------------------------------------------------------------------- # Array container # --------------------------------------------------------------------------- diff --git a/src/rhosocial/activerecord/backend/impl/postgres/functions/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/functions/__init__.py index da294485..3c3dfbc8 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/functions/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/functions/__init__.py @@ -287,10 +287,17 @@ vector_l2_distance, vector_cosine_distance, vector_inner_product, + vector_distance, vector_cosine_similarity, vector_literal, ) +# pgvector similarity-search helpers +from .vector_search import ( + vector_search, + create_vector_index, +) + # PostGIS functions from .postgis import ( st_make_point, @@ -714,8 +721,12 @@ "vector_l2_distance", "vector_cosine_distance", "vector_inner_product", + "vector_distance", "vector_cosine_similarity", "vector_literal", + # pgvector similarity-search helpers + "vector_search", + "create_vector_index", # PostGIS functions - Construction "st_make_point", "st_geom_from_text", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/functions/pgvector.py b/src/rhosocial/activerecord/backend/impl/postgres/functions/pgvector.py index 4bbb2df1..00e88d31 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/functions/pgvector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/functions/pgvector.py @@ -13,6 +13,10 @@ - <=> : Cosine distance - <#> : Inner product (negative) +NOTE: The operator factories treat plain string arguments as literal values, +NOT column references. Pass ``Column(dialect, name)`` (or use ``vector_distance``) +to reference a vector column. + The vector type requires the pgvector extension: CREATE EXTENSION IF NOT EXISTS vector; """ @@ -27,6 +31,13 @@ from rhosocial.activerecord.backend.dialect import SQLDialectBase +_METRIC_OPERATORS = { + "cosine": "<=>", + "l2": "<->", + "ip": "<#>", +} + + def _convert_to_expression( dialect: "SQLDialectBase", expr: Union[PostgresVector, List[float], str, "bases.BaseExpression"], @@ -71,7 +82,8 @@ def vector_l2_distance( Args: dialect: The SQL dialect instance - left: Left operand (column name, vector, or expression) + left: Left operand (vector, string literal, or expression); plain + strings are literal values, use ``Column(dialect, name)`` for columns right: Right operand (vector literal, or expression) Returns: @@ -96,7 +108,8 @@ def vector_cosine_distance( Args: dialect: The SQL dialect instance - left: Left operand (column name, vector, or expression) + left: Left operand (vector, string literal, or expression); plain + strings are literal values, use ``Column(dialect, name)`` for columns right: Right operand (vector literal, or expression) Returns: @@ -119,7 +132,8 @@ def vector_inner_product( Args: dialect: The SQL dialect instance - left: Left operand (column name, vector, or expression) + left: Left operand (vector, string literal, or expression); plain + strings are literal values, use ``Column(dialect, name)`` for columns right: Right operand (vector literal, or expression) Returns: @@ -130,6 +144,41 @@ def vector_inner_product( return BinaryArithmeticExpression(dialect, "<#>", left_expr, right_expr) +def vector_distance( + dialect: "SQLDialectBase", + column: Union[str, "bases.BaseExpression"], + query_vector: Union[PostgresVector, List[float], str, "bases.BaseExpression"], + metric: str = "cosine", +) -> BinaryArithmeticExpression: + """Distance operator for a vector column, dispatching by metric. + + Unlike the raw operator factories, a plain string ``column`` is wrapped in + ``Column`` (a column reference), so passing a column name is safe. + + Args: + dialect: The SQL dialect instance + column: Vector column name (string) or column expression + query_vector: The query vector (``PostgresVector``, ``List[float]``, + str, or expression) + metric: Distance metric - 'cosine' | 'l2' | 'ip' + + Returns: + BinaryArithmeticExpression for the requested distance + """ + if metric not in _METRIC_OPERATORS: + raise ValueError( + f"Unsupported vector metric '{metric}'; expected one of " + f"{tuple(_METRIC_OPERATORS)}" + ) + column_expr = ( + core.Column(dialect, column) if isinstance(column, str) else column + ) + return BinaryArithmeticExpression( + dialect, _METRIC_OPERATORS[metric], + column_expr, _convert_to_expression(dialect, query_vector), + ) + + # === Similarity === def vector_cosine_similarity( @@ -187,6 +236,7 @@ def vector_literal( "vector_l2_distance", "vector_cosine_distance", "vector_inner_product", + "vector_distance", "vector_cosine_similarity", "vector_literal", ] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/functions/vector_search.py b/src/rhosocial/activerecord/backend/impl/postgres/functions/vector_search.py new file mode 100644 index 00000000..44e856b1 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/functions/vector_search.py @@ -0,0 +1,216 @@ +# src/rhosocial/activerecord/backend/impl/postgres/functions/vector_search.py +""" +PostgreSQL pgvector similarity-search helpers. + +High-level helpers built on top of the pgvector operator factories in +``functions/pgvector.py``: + +- ``vector_search``: assemble a complete similarity-search + ``QueryExpression`` (SELECT distance/similarity, WHERE filters, + ORDER BY distance, LIMIT top_k). +- ``create_vector_index``: assemble a ``CreateIndexExpression`` for + HNSW/IVFFlat with metric-to-operator-class mapping. + +They target the RAG knowledge-base pattern: + index(chunk/tenant/meta/vector) + search(query_vector, top_k, filters) + +Requires the pgvector extension: CREATE EXTENSION IF NOT EXISTS vector; +""" + +from typing import List, Optional, Union, Dict, TYPE_CHECKING + +from rhosocial.activerecord.backend.expression import ( + Column, + Literal, + QueryExpression, + TableExpression, +) +from rhosocial.activerecord.backend.expression.core import ( + BaseExpression, + WildcardExpression, +) +from rhosocial.activerecord.backend.expression.operators import BinaryArithmeticExpression +from rhosocial.activerecord.backend.expression.query_parts import ( + LimitOffsetClause, + OrderByClause, +) +from rhosocial.activerecord.backend.expression.query_parts import ( + SQLPredicate, + WhereClause, +) +from rhosocial.activerecord.backend.expression.statements.ddl_index import ( + CreateIndexExpression, +) + +if TYPE_CHECKING: + from rhosocial.activerecord.backend.dialect import SQLDialectBase + +from .pgvector import ( + vector_cosine_distance, + vector_inner_product, + vector_l2_distance, +) + +__all__ = ["vector_search", "create_vector_index"] + +VECTOR_METRICS = ("cosine", "l2", "ip") + +_METRIC_DISTANCE_FN: Dict[str, object] = { + "cosine": vector_cosine_distance, + "l2": vector_l2_distance, + "ip": vector_inner_product, +} + +_METRIC_DISTANCE_ALIAS = { + "cosine": "cosine_distance", + "l2": "l2_distance", + "ip": "inner_product", +} + +_METRIC_SIMILARITY_ALIAS = { + "cosine": "cosine_similarity", + "l2": "l2_similarity", +} + +_METRIC_OPCLASS = { + "cosine": "vector_cosine_ops", + "l2": "vector_l2_ops", + "ip": "vector_ip_ops", +} + + +def _validate_metric(metric: str) -> None: + if metric not in VECTOR_METRICS: + raise ValueError( + f"Unsupported vector metric '{metric}'; expected one of {VECTOR_METRICS}" + ) + + +def vector_search( + dialect: "SQLDialectBase", + table: str, + query_vector, + metric: str = "cosine", + top_k: int = 10, + *, + vector_column: str = "embedding", + columns: Optional[List[Union[str, BaseExpression]]] = None, + where: Optional[Union[SQLPredicate, WhereClause]] = None, + include_distance: bool = True, + include_similarity: bool = False, +) -> QueryExpression: + """Build a pgvector similarity-search query. + + Args: + dialect: The SQL dialect instance + table: Table name to search + query_vector: Query vector (``PostgresVector``, ``List[float]``, str, + or expression) + metric: Distance metric - 'cosine' | 'l2' | 'ip' + top_k: Number of nearest neighbors to return (LIMIT) + vector_column: Vector column name (default 'embedding') + columns: Select list; strings become columns, expressions pass through. + Defaults to ``SELECT *``. + where: Optional filter predicate (e.g. tenant/permission scoping) + include_distance: Include an aliased distance column in SELECT + include_similarity: Include ``1 - distance`` for cosine/l2 in SELECT + + Returns: + A ready-to-execute ``QueryExpression``. + """ + _validate_metric(metric) + if top_k <= 0: + raise ValueError("top_k must be positive") + + vector_col = Column(dialect, vector_column) + distance_expr = _METRIC_DISTANCE_FN[metric](dialect, vector_col, query_vector) + + if columns is None: + select: List[BaseExpression] = [WildcardExpression(dialect)] + else: + select = [ + Column(dialect, c) if isinstance(c, str) else c for c in columns + ] + + if include_distance: + select.append(distance_expr.as_(_METRIC_DISTANCE_ALIAS[metric])) + if include_similarity and metric in _METRIC_SIMILARITY_ALIAS: + similarity = BinaryArithmeticExpression( + dialect, "-", Literal(dialect, 1), distance_expr + ) + select.append(similarity.as_(_METRIC_SIMILARITY_ALIAS[metric])) + + return QueryExpression( + dialect=dialect, + select=select, + from_=TableExpression(dialect, table), + where=where, + order_by=OrderByClause(dialect, [distance_expr]), + limit_offset=LimitOffsetClause(dialect, limit=top_k), + ) + + +def create_vector_index( + dialect: "SQLDialectBase", + table_name: str, + column_name: str = "embedding", + metric: str = "cosine", + index_type: str = "hnsw", + *, + index_name: Optional[str] = None, + m: Optional[int] = None, + ef_construction: Optional[int] = None, + lists: Optional[int] = None, + if_not_exists: bool = True, +) -> CreateIndexExpression: + """Build a pgvector index creation expression. + + Args: + dialect: The SQL dialect instance + table_name: Table name + column_name: Vector column name (default 'embedding') + metric: Distance metric - 'cosine' | 'l2' | 'ip' (maps to opclass) + index_type: Index type - 'hnsw' | 'ivfflat' + index_name: Optional index name (auto-generated if not provided) + m: HNSW max connections per layer + ef_construction: HNSW ef_construction + lists: IVFFlat number of lists + if_not_exists: Add IF NOT EXISTS + + Returns: + A ready-to-execute ``CreateIndexExpression``. + + Note: + IVFFlat requires the table to contain data before the index can be + created (HNSW has no such requirement). Create the index after + inserting rows when using ``index_type='ivfflat'``. + """ + _validate_metric(metric) + index_type_l = index_type.lower() + if index_type_l not in ("hnsw", "ivfflat"): + raise ValueError(f"Unsupported vector index type '{index_type}'; " + "expected 'hnsw' or 'ivfflat'") + + idx_name = index_name or f"idx_{table_name}_{column_name}_{index_type_l}" + with_options: Dict[str, int] = {} + if index_type_l == "hnsw": + if m is not None: + with_options["m"] = m + if ef_construction is not None: + with_options["ef_construction"] = ef_construction + else: + if lists is not None: + with_options["lists"] = lists + + return CreateIndexExpression( + dialect=dialect, + index_name=idx_name, + table_name=table_name, + columns=[column_name], + index_type=index_type_l.upper(), + if_not_exists=if_not_exists, + dialect_options={ + "opclasses": {column_name: _METRIC_OPCLASS[metric]}, + "with": with_options, + }, + ) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/extensions/pgvector.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/extensions/pgvector.py index 08dd6985..7b33b05f 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/extensions/pgvector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/extensions/pgvector.py @@ -43,6 +43,7 @@ def format_create_vector_index_statement( ef_construction: Optional[int] = None, lists: Optional[int] = None, schema: Optional[str] = None, + opclass: str = "vector_cosine_ops", ) -> Tuple[str, tuple]: """Format CREATE INDEX statement for vector column. @@ -55,6 +56,9 @@ def format_create_vector_index_statement( ef_construction: HNSW ef_construction parameter lists: IVFFlat number of lists schema: Optional schema name + opclass: pgvector operator class for the index; defaults to + ``vector_cosine_ops``. Use ``vector_l2_ops`` or ``vector_ip_ops`` + for L2/inner-product distance indexes. Returns: Tuple of (SQL statement, parameters) @@ -70,13 +74,13 @@ def format_create_vector_index_statement( with_clause = f" WITH ({', '.join(with_clauses)})" if with_clauses else "" sql = ( f"CREATE INDEX {index_name} ON {full_table} " - f"USING hnsw ({column_name} vector_cosine_ops){with_clause}" + f"USING hnsw ({column_name} {opclass}){with_clause}" ) else: # ivfflat lists_clause = f" WITH (lists = {lists})" if lists else "" sql = ( f"CREATE INDEX {index_name} ON {full_table} " - f"USING ivfflat ({column_name} vector_cosine_ops){lists_clause}" + f"USING ivfflat ({column_name} {opclass}){lists_clause}" ) return (sql, ()) @@ -88,6 +92,7 @@ def format_create_hnsw_index_statement( index_name: Optional[str] = None, m: Optional[int] = None, ef_construction: Optional[int] = None, + opclass: str = "vector_cosine_ops", ) -> str: """Format CREATE INDEX statement with HNSW index for vector column. @@ -97,6 +102,8 @@ def format_create_hnsw_index_statement( index_name: Optional index name (auto-generated if not provided) m: HNSW M parameter (max connections per layer) ef_construction: HNSW ef_construction parameter + opclass: pgvector operator class for the index; defaults to + ``vector_cosine_ops``. Returns: SQL CREATE INDEX statement @@ -110,5 +117,5 @@ def format_create_hnsw_index_statement( with_clause = f" WITH ({', '.join(with_clauses)})" if with_clauses else "" return ( f"CREATE INDEX {idx_name} ON {table_name} " - f"USING hnsw ({column_name} vector_cosine_ops){with_clause}" + f"USING hnsw ({column_name} {opclass}){with_clause}" ) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/types/data_type_formatting.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/types/data_type_formatting.py index 73c155f6..2cfb4833 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/types/data_type_formatting.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/types/data_type_formatting.py @@ -80,6 +80,8 @@ PostgresUUIDType, PostgresVarBitType, PostgresVectorType, + PostgresHalfvecType, + PostgresSparsevecType, PostgresXID8Type, PostgresXIDType, PostgresXMLType, @@ -307,6 +309,14 @@ def format_data_type_geography(self, data_type) -> Tuple[str, tuple]: def format_data_type_vector(self, data_type) -> Tuple[str, tuple]: return f"VECTOR({data_type.dim})", () + @DDLTypeMixin.handles(PostgresHalfvecType) + def format_data_type_halfvec(self, data_type) -> Tuple[str, tuple]: + return f"HALFVEC({data_type.dim})", () + + @DDLTypeMixin.handles(PostgresSparsevecType) + def format_data_type_sparsevec(self, data_type) -> Tuple[str, tuple]: + return f"SPARSEVEC({data_type.dim})", () + @DDLTypeMixin.handles(PostgresCitextType) def format_data_type_citext(self, data_type) -> Tuple[str, tuple]: return "CITEXT", () @@ -809,10 +819,27 @@ def parse_type(self, raw: str) -> DataType: if upper.startswith("GEOMETRY"): return PostgresGeometryType() - # Vector type (pgvector) - if upper.startswith("VECTOR"): + # Vector types (pgvector) + vector_match = re.match(r"^(VECTOR|HALFVEC|SPARSEVEC)\b", upper) + if vector_match: + if ( + hasattr(self, "_extensions") + and "vector" in self._extensions + and not self._extensions["vector"].installed + ): + raise RuntimeError( + "Cannot use pgvector types: the 'vector' (pgvector) extension " + "is not installed. Run: CREATE EXTENSION IF NOT EXISTS vector;" + ) nums = re.findall(r"\d+", stripped) dim = int(nums[0]) if nums else 0 + vector_kind = vector_match.group(1) + if vector_kind == "HALFVEC": + from ...expression.types import PostgresHalfvecType + return PostgresHalfvecType(dim) + if vector_kind == "SPARSEVEC": + from ...expression.types import PostgresSparsevecType + return PostgresSparsevecType(dim) from ...expression.types import PostgresVectorType return PostgresVectorType(dim) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/protocols/extensions/pgvector.py b/src/rhosocial/activerecord/backend/impl/postgres/protocols/extensions/pgvector.py index f208adf7..53146cee 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/protocols/extensions/pgvector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/protocols/extensions/pgvector.py @@ -60,6 +60,7 @@ def format_create_vector_index_statement( ef_construction: Optional[int] = None, lists: Optional[int] = None, schema: Optional[str] = None, + opclass: str = "vector_cosine_ops", ) -> Tuple[str, tuple]: """Format CREATE INDEX statement for vector column.""" ... @@ -71,6 +72,7 @@ def format_create_hnsw_index_statement( index_name: Optional[str] = None, m: Optional[int] = None, ef_construction: Optional[int] = None, + opclass: str = "vector_cosine_ops", ) -> str: """Format CREATE INDEX statement with HNSW index for vector column.""" ... diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/examples_compile_test.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/examples_compile_test.py new file mode 100644 index 00000000..837afc53 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/examples_compile_test.py @@ -0,0 +1,36 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/examples_compile_test.py +"""Guard against syntax/API drift in the runnable examples. + +Every script under ``examples/`` must at least compile. This test runs +without a database and fails fast if an example introduces a syntax error. +(API drift that only manifests at runtime is caught by ``scripts/run_examples.py`` +against a live database in CI.) +""" +import py_compile +from pathlib import Path + +import pytest + +EXAMPLES_ROOT = ( + Path(__file__).resolve().parents[5] + / "src" / "rhosocial" / "activerecord" / "backend" / "impl" / "postgres" + / "examples" +) + + +def _collect_examples(): + if not EXAMPLES_ROOT.exists(): + return [] + return sorted(EXAMPLES_ROOT.rglob("*.py")) + + +def test_all_examples_compile(): + examples = _collect_examples() + assert examples, f"no examples found under {EXAMPLES_ROOT}" + failures = [] + for path in examples: + try: + py_compile.compile(str(path), doraise=True) + except py_compile.PyCompileError as exc: # pragma: no cover + failures.append(str(exc)) + assert not failures, "example compile failures:\n" + "\n".join(failures) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py new file mode 100644 index 00000000..e0e8a3fb --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py @@ -0,0 +1,142 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py +"""SQL-snapshot tests for the pgvector similarity-search helpers. + +These tests build expressions with a bare ``PostgresDialect`` (no DB +connection) and assert on the generated SQL, so they run without a server. +""" +import pytest + +from rhosocial.activerecord.backend.expression import Column, Literal, predicates +from rhosocial.activerecord.backend.impl.postgres.functions import ( + create_vector_index, + vector_distance, + vector_search, +) + + +class TestVectorSearch: + """Tests for the vector_search helper.""" + + def test_default_select_star(self, postgres_dialect): + expr = vector_search(postgres_dialect, "documents", [1.0, 0.5, 0.2]) + sql, params = expr.to_sql() + assert "SELECT *" in sql + assert 'AS "cosine_distance"' in sql + assert "ORDER BY" in sql and "<=>" in sql + assert "LIMIT %s" in sql + assert params[-1] == 10 + + def test_columns_and_similarity(self, postgres_dialect): + expr = vector_search( + postgres_dialect, + "documents", + [1.0, 0.55, 0.18], + columns=["content", "tenant"], + include_similarity=True, + top_k=3, + ) + sql, params = expr.to_sql() + assert "SELECT \"content\", \"tenant\"" in sql + assert 'AS "cosine_distance"' in sql + assert 'AS "cosine_similarity"' in sql + assert "LIMIT %s" in sql and params[-1] == 3 + + def test_where_filter(self, postgres_dialect): + tenant_pred = predicates.ComparisonPredicate( + postgres_dialect, "=", Column(postgres_dialect, "tenant"), + Literal(postgres_dialect, "acme"), + ) + expr = vector_search( + postgres_dialect, "documents", [1.0, 0.5, 0.2], + where=tenant_pred, + ) + sql, params = expr.to_sql() + assert 'WHERE "tenant" = %s' in sql + assert "acme" in params + + def test_l2_metric(self, postgres_dialect): + expr = vector_search( + postgres_dialect, "documents", [0.1, 0.2, 0.9], + metric="l2", columns=["content"], + ) + sql, params = expr.to_sql() + assert 'AS "l2_distance"' in sql + assert "<->" in sql + + def test_ip_metric(self, postgres_dialect): + expr = vector_search( + postgres_dialect, "documents", [0.1, 0.2, 0.9], + metric="ip", columns=["content"], + ) + sql, params = expr.to_sql() + assert 'AS "inner_product"' in sql + assert "<#>" in sql + + def test_invalid_metric(self, postgres_dialect): + with pytest.raises(ValueError, match="Unsupported vector metric"): + vector_search(postgres_dialect, "documents", [1.0], metric="hamming") + + def test_non_positive_top_k(self, postgres_dialect): + with pytest.raises(ValueError, match="top_k must be positive"): + vector_search(postgres_dialect, "documents", [1.0], top_k=0) + + +class TestVectorDistance: + """Tests for the vector_distance column-name convenience.""" + + def test_column_name_wrapped_as_column(self, postgres_dialect): + expr = vector_distance(postgres_dialect, "embedding", [1.0, 0.5, 0.2]) + sql, params = expr.to_sql() + # Column name must render as identifier, not a string literal parameter. + assert 'AS' not in sql + assert '"embedding" <=> %s::vector(3)' in sql + assert "embedding" not in params + + def test_l2_column(self, postgres_dialect): + expr = vector_distance(postgres_dialect, "embedding", [1.0], metric="l2") + sql, params = expr.to_sql() + assert '"embedding" <-> %s::vector(1)' in sql + + def test_invalid_metric(self, postgres_dialect): + with pytest.raises(ValueError, match="Unsupported vector metric"): + vector_distance(postgres_dialect, "embedding", [1.0], metric="bad") + + +class TestCreateVectorIndex: + """Tests for the create_vector_index helper.""" + + def test_hnsw_default_cosine(self, postgres_dialect): + expr = create_vector_index(postgres_dialect, "documents", "embedding") + sql, params = expr.to_sql() + assert "CREATE INDEX IF NOT EXISTS" in sql + assert 'USING HNSW ("embedding" vector_cosine_ops)' in sql + + def test_hnsw_with_options(self, postgres_dialect): + expr = create_vector_index( + postgres_dialect, "documents", "embedding", + index_type="hnsw", m=16, ef_construction=64, + ) + sql, params = expr.to_sql() + assert "WITH (m = 16, ef_construction = 64)" in sql + + def test_ivfflat_l2(self, postgres_dialect): + expr = create_vector_index( + postgres_dialect, "documents", "embedding", + metric="l2", index_type="ivfflat", lists=4, + ) + sql, params = expr.to_sql() + assert 'USING IVFFLAT ("embedding" vector_l2_ops)' in sql + assert "WITH (lists = 4)" in sql + + def test_ip_opclass(self, postgres_dialect): + expr = create_vector_index( + postgres_dialect, "documents", "embedding", + metric="ip", index_type="hnsw", + ) + sql, params = expr.to_sql() + assert 'vector_ip_ops' in sql + + def test_invalid_index_type(self, postgres_dialect): + with pytest.raises(ValueError, match="Unsupported vector index type"): + create_vector_index(postgres_dialect, "documents", "embedding", + index_type="btree") From f4c8a523247ff9734d3857a5d1716e316b826e10 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 29 Aug 2026 00:28:30 +0800 Subject: [PATCH 28/31] test: add targeted coverage for low-coverage patch areas - extend PostgresListAdapter string parsing tests (JSON + PG array literal) - add HALFVEC/SPARSEVEC DDL format/parse and eq/hash coverage - cover vector_search edge cases (ip similarity, index options, invalid metric) - add _get_default_schema config-aware resolution tests for both introspection stacks - cover backend autocommit handler branches (sync + async, True/False) - add CLI --ssl mapping and --async flag tests - add CLI blackbox tests for introspect/status --async fix(cli): correct async introspect method names (remove _async suffix) The _handle_introspect_async path called nonexistent list_tables_async, list_views_async etc. Async introspector methods use the same names as sync (no _async suffix). Fixes broken async introspect branch. --- .../backend/impl/postgres/cli/introspect.py | 16 ++-- .../backend/expression/vector_search_test.py | 56 +++++++++++++ .../test_introspection_default_schema.py | 84 +++++++++++++++++++ .../postgres/test_connection_config.py | 82 ++++++++++++++++++ .../feature/backend/test_adapters.py | 45 ++++++++++ .../backend/test_cli_connection_args.py | 46 ++++++++++ .../types/test_data_type_formatting.py | 56 +++++++++++++ .../feature/backend/cli/test_cli_blackbox.py | 17 ++++ 8 files changed, 394 insertions(+), 8 deletions(-) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_cli_connection_args.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index c8c11c7c..d5d42a91 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -280,7 +280,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider introspector = backend.introspector if args.type == "tables": - tables = await introspector.list_tables_async( + tables = await introspector.list_tables( schema=args.schema, include_system=args.include_system ) @@ -288,7 +288,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider provider.display_results(data, title="Tables") elif args.type == "views": - views = await introspector.list_views_async(schema=args.schema) + views = await introspector.list_views(schema=args.schema) data = _serialize_for_output(views) provider.display_results(data, title="Views") @@ -296,7 +296,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider if not args.name: print("Error: Table name is required for 'table' introspection", file=sys.stderr) sys.exit(1) - info = await introspector.get_table_info_async(args.name, schema=args.schema) + info = await introspector.get_table_info(args.name, schema=args.schema) if info: provider.display_results(_serialize_for_output(info.columns), title=f"Columns of {args.name}") if info.indexes: @@ -314,7 +314,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider if not args.name: print("Error: Table name is required for 'columns' introspection", file=sys.stderr) sys.exit(1) - columns = await introspector.list_columns_async(args.name, schema=args.schema) + columns = await introspector.list_columns(args.name, schema=args.schema) data = _serialize_for_output(columns) provider.display_results(data, title=f"Columns of {args.name}") @@ -322,7 +322,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider if not args.name: print("Error: Table name is required for 'indexes' introspection", file=sys.stderr) sys.exit(1) - indexes = await introspector.list_indexes_async(args.name, schema=args.schema) + indexes = await introspector.list_indexes(args.name, schema=args.schema) data = _serialize_for_output(indexes) provider.display_results(data, title=f"Indexes of {args.name}") @@ -330,12 +330,12 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider if not args.name: print("Error: Table name is required for 'foreign-keys' introspection", file=sys.stderr) sys.exit(1) - fks = await introspector.list_foreign_keys_async(args.name, schema=args.schema) + fks = await introspector.list_foreign_keys(args.name, schema=args.schema) data = _serialize_for_output(fks) provider.display_results(data, title=f"Foreign Keys of {args.name}") elif args.type == "triggers": - triggers = await introspector.list_triggers_async( + triggers = await introspector.list_triggers( table_name=args.name, schema=args.schema ) @@ -343,7 +343,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider provider.display_results(data, title="Triggers") elif args.type == "database": - info = await introspector.get_database_info_async() + info = await introspector.get_database_info() data = _serialize_for_output(info) provider.display_results([data], title="Database Info") diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py index e0e8a3fb..54f7fbd4 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/expression/vector_search_test.py @@ -72,6 +72,32 @@ def test_ip_metric(self, postgres_dialect): assert 'AS "inner_product"' in sql assert "<#>" in sql + def test_ip_metric_with_similarity_requested(self, postgres_dialect): + """ip metric has no 1-distance similarity alias, so none is added.""" + expr = vector_search( + postgres_dialect, "documents", [0.1, 0.2, 0.9], + metric="ip", columns=["content"], include_similarity=True, + ) + sql, params = expr.to_sql() + assert 'AS "inner_product"' in sql + assert "similarity" not in sql + + def test_include_distance_false(self, postgres_dialect): + expr = vector_search( + postgres_dialect, "documents", [0.1, 0.2, 0.9], + columns=["content"], include_distance=False, + ) + sql, params = expr.to_sql() + assert "cosine_distance" not in sql + + def test_l2_similarity_alias(self, postgres_dialect): + expr = vector_search( + postgres_dialect, "documents", [0.1, 0.2, 0.9], + metric="l2", include_similarity=True, + ) + sql, params = expr.to_sql() + assert 'AS "l2_similarity"' in sql + def test_invalid_metric(self, postgres_dialect): with pytest.raises(ValueError, match="Unsupported vector metric"): vector_search(postgres_dialect, "documents", [1.0], metric="hamming") @@ -128,6 +154,31 @@ def test_ivfflat_l2(self, postgres_dialect): assert 'USING IVFFLAT ("embedding" vector_l2_ops)' in sql assert "WITH (lists = 4)" in sql + def test_ivfflat_without_lists(self, postgres_dialect): + expr = create_vector_index( + postgres_dialect, "documents", "embedding", + metric="l2", index_type="ivfflat", + ) + sql, params = expr.to_sql() + assert 'USING IVFFLAT ("embedding" vector_l2_ops)' in sql + assert "WITH" not in sql + + def test_custom_index_name(self, postgres_dialect): + expr = create_vector_index( + postgres_dialect, "documents", "embedding", + index_name="doc_emb_hnsw", + ) + sql, params = expr.to_sql() + assert "doc_emb_hnsw" in sql + + def test_auto_index_name(self, postgres_dialect): + expr = create_vector_index( + postgres_dialect, "documents", "embedding", + index_type="hnsw", + ) + sql, params = expr.to_sql() + assert "idx_documents_embedding_hnsw" in sql + def test_ip_opclass(self, postgres_dialect): expr = create_vector_index( postgres_dialect, "documents", "embedding", @@ -136,6 +187,11 @@ def test_ip_opclass(self, postgres_dialect): sql, params = expr.to_sql() assert 'vector_ip_ops' in sql + def test_invalid_metric(self, postgres_dialect): + with pytest.raises(ValueError, match="Unsupported vector metric"): + create_vector_index(postgres_dialect, "documents", "embedding", + metric="hamming") + def test_invalid_index_type(self, postgres_dialect): with pytest.raises(ValueError, match="Unsupported vector index type"): create_vector_index(postgres_dialect, "documents", "embedding", diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py new file mode 100644 index 00000000..54e20116 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py @@ -0,0 +1,84 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py +"""Offline tests for ``_get_default_schema`` config-aware resolution. + +Both introspection stacks gained config-aware default-schema resolution: + +1. ``PostgreSQLIntrospectorMixin._get_default_schema`` (introspector) reads + ``self._backend.config``. +2. ``PostgresIntrospectionCapabilityMixin._get_default_schema`` (dialect SQL + generation) reads ``self._config`` or ``self._backend``. + +Priority: config.default_schema > first entry of config.search_path > 'public'. +""" +from rhosocial.activerecord.backend.impl.postgres import PostgresBackend +from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig +from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect +from rhosocial.activerecord.backend.impl.postgres.introspection import ( + SyncPostgreSQLIntrospector, +) + + +def make_config(**overrides) -> PostgresConnectionConfig: + params = dict( + host="localhost", + port=5432, + database="test", + username="postgres", + password="", + ) + params.update(overrides) + return PostgresConnectionConfig(**params) + + +def make_introspector(config: PostgresConnectionConfig) -> SyncPostgreSQLIntrospector: + backend = PostgresBackend(connection_config=config) + return SyncPostgreSQLIntrospector(backend, executor=object()) + + +class TestIntrospectorDefaultSchema: + """PostgreSQLIntrospectorMixin._get_default_schema (self._backend.config).""" + + def test_returns_public_when_not_configured(self): + introspector = make_introspector(make_config()) + assert introspector._get_default_schema() == "public" + + def test_returns_default_schema_from_config(self): + introspector = make_introspector(make_config(default_schema="broker")) + assert introspector._get_default_schema() == "broker" + + def test_returns_first_entry_of_search_path(self): + introspector = make_introspector(make_config(search_path="broker, public")) + assert introspector._get_default_schema() == "broker" + + def test_strips_quotes_from_search_path_entry(self): + introspector = make_introspector(make_config(search_path='"broker", public')) + assert introspector._get_default_schema() == "broker" + + def test_default_schema_takes_priority_over_search_path(self): + introspector = make_introspector( + make_config(default_schema="app", search_path="broker, public") + ) + assert introspector._get_default_schema() == "app" + + +class TestDialectMixinDefaultSchema: + """PostgresIntrospectionCapabilityMixin._get_default_schema (dialect).""" + + def test_returns_public_when_not_configured(self): + assert PostgresDialect()._get_default_schema() == "public" + + def test_returns_default_schema_from_config(self): + dialect = PostgresDialect() + dialect._config = make_config(default_schema="broker") + assert dialect._get_default_schema() == "broker" + + def test_returns_first_entry_of_search_path(self): + dialect = PostgresDialect() + dialect._config = make_config(search_path="broker, public") + assert dialect._get_default_schema() == "broker" + + def test_returns_schema_from_backend(self): + dialect = PostgresDialect() + backend = PostgresBackend(connection_config=make_config(search_path="warehouse")) + dialect._backend = backend + assert dialect._get_default_schema() == "warehouse" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py index 54c7c404..335b57e9 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_connection_config.py @@ -21,6 +21,28 @@ ) +class FakeConnection: + def __init__(self): + self.commit_count = 0 + + def commit(self): + self.commit_count += 1 + + def rollback(self): + pass + + +class FakeAsyncConnection: + def __init__(self): + self.commit_count = 0 + + async def commit(self): + self.commit_count += 1 + + async def rollback(self): + pass + + def make_config(**overrides) -> PostgresConnectionConfig: params = dict( host="localhost", @@ -70,3 +92,63 @@ async def test_async_backend_requires_commit_by_default(self): async def test_async_backend_skips_commit_when_autocommit(self): backend = AsyncPostgresBackend(connection_config=make_config(autocommit=True)) assert backend.requires_manual_commit() is False + + +class TestAutoCommitHandlers: + """The backend auto-commit handlers honor config.autocommit.""" + + def test_handle_auto_commit_commits_when_disabled(self): + backend = PostgresBackend(connection_config=make_config()) + backend._connection = FakeConnection() + backend._handle_auto_commit() + assert backend._connection.commit_count == 1 + + def test_handle_auto_commit_skips_when_enabled(self): + backend = PostgresBackend(connection_config=make_config(autocommit=True)) + backend._connection = FakeConnection() + backend._handle_auto_commit() + assert backend._connection.commit_count == 0 + + def test_handle_auto_commit_noop_without_connection(self): + backend = PostgresBackend(connection_config=make_config()) + backend._handle_auto_commit() + + def test_handle_auto_commit_if_needed_commits_when_disabled(self): + backend = PostgresBackend(connection_config=make_config()) + backend._connection = FakeConnection() + backend._handle_auto_commit_if_needed() + assert backend._connection.commit_count == 1 + + def test_handle_auto_commit_if_needed_skips_when_enabled(self): + backend = PostgresBackend(connection_config=make_config(autocommit=True)) + backend._connection = FakeConnection() + backend._handle_auto_commit_if_needed() + assert backend._connection.commit_count == 0 + + @pytest.mark.asyncio + async def test_async_handle_auto_commit_commits_when_disabled(self): + backend = AsyncPostgresBackend(connection_config=make_config()) + backend._connection = FakeAsyncConnection() + await backend._handle_auto_commit() + assert backend._connection.commit_count == 1 + + @pytest.mark.asyncio + async def test_async_handle_auto_commit_skips_when_enabled(self): + backend = AsyncPostgresBackend(connection_config=make_config(autocommit=True)) + backend._connection = FakeAsyncConnection() + await backend._handle_auto_commit() + assert backend._connection.commit_count == 0 + + @pytest.mark.asyncio + async def test_async_handle_auto_commit_if_needed_commits_when_disabled(self): + backend = AsyncPostgresBackend(connection_config=make_config()) + backend._connection = FakeAsyncConnection() + await backend._handle_auto_commit_if_needed() + assert backend._connection.commit_count == 1 + + @pytest.mark.asyncio + async def test_async_handle_auto_commit_if_needed_skips_when_enabled(self): + backend = AsyncPostgresBackend(connection_config=make_config(autocommit=True)) + backend._connection = FakeAsyncConnection() + await backend._handle_auto_commit_if_needed() + assert backend._connection.commit_count == 0 diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py index 22de8241..639e509e 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_adapters.py @@ -171,10 +171,55 @@ def test_roundtrip(self, enum_a): assert enum_a.from_database(s, str) == s class TestList: + def test_to_database(self, list_a): + arr = [1, 2, 3] + assert list_a.to_database(arr, list) == arr + def test_roundtrip(self, list_a): arr = [1, 2, 3] assert list_a.from_database(arr, list) == arr + def test_from_database_json_string(self, list_a): + assert list_a.from_database("[1, 2, 3]", list) == [1, 2, 3] + + def test_from_database_json_string_with_whitespace(self, list_a): + assert list_a.from_database(' ["a", "b"] ', list) == ["a", "b"] + + def test_from_database_pg_array_literal(self, list_a): + assert list_a.from_database("{1,2,3}", list) == ["1", "2", "3"] + + def test_from_database_pg_array_literal_quoted(self, list_a): + assert list_a.from_database('{"a","b c"}', list) == ["a", "b c"] + + def test_from_database_pg_array_literal_null(self, list_a): + assert list_a.from_database("{1,NULL,3}", list) == ["1", None, "3"] + + def test_from_database_invalid_string_raises(self, list_a): + with pytest.raises(TypeError, match="Cannot convert str to list"): + list_a.from_database("not a list", list) + + def test_from_database_invalid_json_string_fallthrough(self, list_a): + with pytest.raises(TypeError, match="Cannot convert str to list"): + list_a.from_database("[1, 2", list) + + def test_from_database_invalid_array_literal_fallthrough(self, list_a): + with pytest.raises(TypeError, match="Cannot convert str to list"): + list_a.from_database("{}}", list) # extra brace triggers DataError + + def test_from_database_non_string_raises(self, list_a): + with pytest.raises(TypeError, match="Cannot convert int to list"): + list_a.from_database(123, list) + + def test_parse_array_literal_invalid_raises(self): + with pytest.raises(ValueError, match="Not a PostgreSQL array literal"): + PostgresListAdapter._parse_array_literal("not-an-array") + + def test_to_database_batch(self, list_a): + assert list_a.to_database_batch([[1], [2, 3]], list) == [[1], [2, 3]] + + def test_from_database_batch(self, list_a): + assert list_a.from_database_batch([[1], [2, 3]], list) == [[1], [2, 3]] + class TestVector: def test_roundtrip(self, vec): v = [1.0, 2.5, 3.0] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_cli_connection_args.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_cli_connection_args.py new file mode 100644 index 00000000..fccee4ec --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_cli_connection_args.py @@ -0,0 +1,46 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_cli_connection_args.py +"""Offline tests for ``resolve_connection_config_from_args`` (SSL mapping, etc.).""" +import argparse + +from rhosocial.activerecord.backend.impl.postgres.cli.connection import ( + add_connection_args, + resolve_connection_config_from_args, +) + + +def _parse(extra=None): + parser = argparse.ArgumentParser() + add_connection_args(parser) + return parser.parse_args(extra or []) + + +class TestCLISSLMapping: + """SSL parameter mapping in resolve_connection_config_from_args.""" + + def test_default_ssl_maps_to_prefer(self): + config = resolve_connection_config_from_args(_parse()) + assert config.sslmode == "prefer" + + def test_ssl_auto(self): + config = resolve_connection_config_from_args(_parse(["--ssl", "auto"])) + assert config.sslmode == "prefer" + + def test_ssl_require(self): + config = resolve_connection_config_from_args(_parse(["--ssl", "require"])) + assert config.sslmode == "require" + + def test_ssl_verify_ca(self): + config = resolve_connection_config_from_args(_parse(["--ssl", "verify-ca"])) + assert config.sslmode == "verify-ca" + + def test_ssl_verify_full(self): + config = resolve_connection_config_from_args(_parse(["--ssl", "verify-full"])) + assert config.sslmode == "verify-full" + + def test_ssl_disabled(self): + config = resolve_connection_config_from_args(_parse(["--ssl", "disabled"])) + assert config.sslmode == "disable" + + def test_is_async_flag(self): + assert _parse(["--async"]).is_async is True + assert _parse([]).is_async is False \ No newline at end of file diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/types/test_data_type_formatting.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/types/test_data_type_formatting.py index c7c286f9..ef74389f 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/types/test_data_type_formatting.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/types/test_data_type_formatting.py @@ -86,6 +86,8 @@ PostgresTsTzRangeType, PostgresUUIDType, PostgresVarBitType, + PostgresHalfvecType, + PostgresSparsevecType, PostgresVectorType, PostgresXID8Type, PostgresXIDType, @@ -152,6 +154,8 @@ def dialect(): (PostgresGeometryType(), "GEOMETRY"), (PostgresGeographyType(), "GEOGRAPHY"), (PostgresVectorType(dim=384), "VECTOR(384)"), + (PostgresHalfvecType(dim=128), "HALFVEC(128)"), + (PostgresSparsevecType(dim=16), "SPARSEVEC(16)"), (PostgresCitextType(), "CITEXT"), (PostgresCubeType(), "CUBE"), (PostgresLtreeType(), "LTREE"), @@ -346,6 +350,10 @@ def test_format_unregistered_type_raises(dialect): ("GEOMETRY", PostgresGeometryType, {}), ("VECTOR", PostgresVectorType, {"dim": 0}), ("VECTOR(768)", PostgresVectorType, {"dim": 768}), + ("HALFVEC", PostgresHalfvecType, {"dim": 0}), + ("HALFVEC(128)", PostgresHalfvecType, {"dim": 128}), + ("SPARSEVEC", PostgresSparsevecType, {"dim": 0}), + ("SPARSEVEC(16)", PostgresSparsevecType, {"dim": 16}), ] @@ -412,6 +420,54 @@ def test_parse_type_case_insensitive(dialect): assert result.length == 25 +def test_parse_vector_requires_extension(dialect): + """parse_type must reject pgvector types when the extension is not installed.""" + from rhosocial.activerecord.backend.impl.postgres.protocols.base import ( + PostgresExtensionInfo, + ) + dialect._extensions = { + "vector": PostgresExtensionInfo(name="vector", installed=False, available=True) + } + with pytest.raises(RuntimeError, match="vector.*extension"): + dialect.parse_type("VECTOR(3)") + with pytest.raises(RuntimeError, match="vector.*extension"): + dialect.parse_type("HALFVEC(3)") + with pytest.raises(RuntimeError, match="vector.*extension"): + dialect.parse_type("SPARSEVEC(3)") + + +def test_parse_vector_allowed_when_extension_installed(dialect): + from rhosocial.activerecord.backend.impl.postgres.protocols.base import ( + PostgresExtensionInfo, + ) + dialect._extensions = { + "vector": PostgresExtensionInfo(name="vector", installed=True) + } + assert type(dialect.parse_type("VECTOR(3)")) is PostgresVectorType + assert type(dialect.parse_type("HALFVEC(3)")) is PostgresHalfvecType + assert type(dialect.parse_type("SPARSEVEC(3)")) is PostgresSparsevecType + + +def test_vector_types_equality_and_hash(): + assert PostgresVectorType(3) == PostgresVectorType(3) + assert PostgresVectorType(3) != PostgresVectorType(4) + assert PostgresVectorType(3) != PostgresHalfvecType(3) + assert PostgresVectorType(3) != object() + assert hash(PostgresVectorType(3)) == hash(PostgresVectorType(3)) + + assert PostgresHalfvecType(3) == PostgresHalfvecType(3) + assert PostgresHalfvecType(3) != PostgresHalfvecType(4) + assert PostgresHalfvecType(3) != PostgresSparsevecType(3) + assert PostgresHalfvecType(3) != object() + assert hash(PostgresHalfvecType(3)) == hash(PostgresHalfvecType(3)) + + assert PostgresSparsevecType(5) == PostgresSparsevecType(5) + assert PostgresSparsevecType(5) != PostgresSparsevecType(6) + assert PostgresSparsevecType(5) != PostgresVectorType(5) + assert PostgresSparsevecType(5) != object() + assert hash(PostgresSparsevecType(5)) == hash(PostgresSparsevecType(5)) + + # --------------------------------------------------------------------------- # supports_data_types # --------------------------------------------------------------------------- diff --git a/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py b/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py index 134a1374..a3269d55 100644 --- a/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py +++ b/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py @@ -78,6 +78,23 @@ def test_status(self, conn_args): assert exc is None, f"stderr: {err}\nstdout: {out}" assert json.loads(out) + def test_status_async(self, conn_args): + out, err, exc = run_cli(["status"] + conn_args + ["-o", "json", "--async"]) + assert exc is None, f"stderr: {err}\nstdout: {out}" + assert json.loads(out) + + +class TestIntrospect: + def test_introspect(self, conn_args): + out, err, exc = run_cli(["introspect", "tables"] + conn_args + ["-o", "json"]) + assert exc is None, f"stderr: {err}\nstdout: {out}" + assert json.loads(out) + + def test_introspect_async(self, conn_args): + out, err, exc = run_cli(["introspect", "tables"] + conn_args + ["-o", "json", "--async"]) + assert exc is None, f"stderr: {err}\nstdout: {out}" + assert json.loads(out) + class TestInfo: def test_info(self): From 3b04e9aee9011fcf3f4a50b0f6a666c69003f641 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 29 Aug 2026 01:08:15 +0800 Subject: [PATCH 29/31] fix(test): tolerate empty table list in introspect blackbox tests CI runs against a fresh test_db where 'introspect tables' returns [], so assert json.loads(out) (truthy) fails. Assert the parsed JSON is a list instead, which holds whether or not tables exist. --- .../feature/backend/cli/test_cli_blackbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py b/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py index a3269d55..210f89ad 100644 --- a/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py +++ b/tests/rhosocial/activerecord_test/feature/backend/cli/test_cli_blackbox.py @@ -88,12 +88,12 @@ class TestIntrospect: def test_introspect(self, conn_args): out, err, exc = run_cli(["introspect", "tables"] + conn_args + ["-o", "json"]) assert exc is None, f"stderr: {err}\nstdout: {out}" - assert json.loads(out) + assert isinstance(json.loads(out), list) def test_introspect_async(self, conn_args): out, err, exc = run_cli(["introspect", "tables"] + conn_args + ["-o", "json", "--async"]) assert exc is None, f"stderr: {err}\nstdout: {out}" - assert json.loads(out) + assert isinstance(json.loads(out), list) class TestInfo: From c29da9f5547eb0de68f9a823f299c7471fb02694 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 29 Aug 2026 01:17:04 +0800 Subject: [PATCH 30/31] chore: add changelog fragment for PR #55 --- changelog.d/55.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/55.added.md diff --git a/changelog.d/55.added.md b/changelog.d/55.added.md new file mode 100644 index 00000000..851351bc --- /dev/null +++ b/changelog.d/55.added.md @@ -0,0 +1 @@ +Add parallel testing infrastructure and PostgreSQL backend improvements \ No newline at end of file From 7137ccec6d81d80788937540debd7798ba5ab2c4 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 29 Aug 2026 01:18:46 +0800 Subject: [PATCH 31/31] docs: remove non-standard changelog fragment The file name stateflow-cross-backend-issues.fixed.md does not follow the ..md convention. Its content was already merged into the relevant fix commit. --- changelog.d/stateflow-cross-backend-issues.fixed.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/stateflow-cross-backend-issues.fixed.md diff --git a/changelog.d/stateflow-cross-backend-issues.fixed.md b/changelog.d/stateflow-cross-backend-issues.fixed.md deleted file mode 100644 index 91bb53ce..00000000 --- a/changelog.d/stateflow-cross-backend-issues.fixed.md +++ /dev/null @@ -1 +0,0 @@ -- `PostgresListAdapter` now handles string values on read (parses JSON and PostgreSQL array literals via psycopg's own array loader), fixing cross-backend `list` field round-trip with TEXT columns. \ No newline at end of file