From 3225ca6606140ec3ecdd37e75d928a716736abb4 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 29 Mar 2026 11:43:35 +0800 Subject: [PATCH 01/11] chore: bump version to 1.0.0.dev8 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a899e329..8aeec914 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rhosocial_activerecord_postgres" -version = "1.0.0.dev7" +version = "1.0.0.dev8" description = "postgres backend implementation for rhosocial-activerecord, providing a robust and optimized postgres database support." readme = "README.md" license = { file = "LICENSE" } From d042c219166be1ea9ea770072302a98d0c65040a Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 31 Mar 2026 12:26:02 +0800 Subject: [PATCH 02/11] refactor(test): eliminate tests.* imports in introspection tests Replace `from tests.providers.scenarios` and `from tests.rhosocial...` imports with local implementations to match the coding style used in other test files. This ensures tests can run with both `pytest tests/` and `python -m pytest tests/` without path issues. --- .../feature/backend/introspection/conftest.py | 76 +++++++++++++++---- .../introspection/test_introspection_views.py | 35 ++++++++- 2 files changed, 95 insertions(+), 16 deletions(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/conftest.py index 2000a5c4..aa95ee6e 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/conftest.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/conftest.py @@ -6,8 +6,14 @@ functionality with PostgreSQL backends. """ +import os import pytest import pytest_asyncio +import yaml +from typing import Dict, Any, Tuple, Type + +from rhosocial.activerecord.backend.impl.postgres import PostgresBackend, AsyncPostgresBackend +from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig # SQL statements for test tables _TABLES_SQL = [ @@ -145,6 +151,62 @@ ] +# --- Scenario Loading Logic (consistent with feature/backend/conftest.py) --- + +SCENARIO_MAP: Dict[str, Dict[str, Any]] = {} + + +def _load_scenarios_from_config(): + """Load scenarios from configuration file.""" + config_path = None + env_config_path = os.getenv("POSTGRES_SCENARIOS_CONFIG_PATH") + + if env_config_path and os.path.exists(env_config_path): + config_path = env_config_path + else: + default_path = os.path.join(os.path.dirname(__file__), "../../../../../config", "postgres_scenarios.yaml") + if os.path.exists(default_path): + config_path = default_path + + if not config_path: + raise FileNotFoundError( + "No PostgreSQL scenarios configuration file found. " + "Set POSTGRES_SCENARIOS_CONFIG_PATH or place postgres_scenarios.yaml in tests/config." + ) + + with open(config_path, 'r', encoding='utf-8') as f: + config_data = yaml.safe_load(f) + + if 'scenarios' not in config_data: + raise ValueError(f"Configuration file {config_path} does not contain 'scenarios' key") + + for scenario_name, config in config_data['scenarios'].items(): + if config: + SCENARIO_MAP[scenario_name] = config + + +_load_scenarios_from_config() + + +def get_scenario(name: str) -> Tuple[Type[PostgresBackend], PostgresConnectionConfig]: + """Get backend class and connection config for a scenario.""" + if name not in SCENARIO_MAP: + if SCENARIO_MAP: + name = next(iter(SCENARIO_MAP)) + else: + raise ValueError("No scenarios registered") + + config_dict = SCENARIO_MAP[name].copy() + config_dict.pop('enabled', None) + config = PostgresConnectionConfig(**config_dict) + return PostgresBackend, config + + +def get_enabled_scenarios() -> Dict[str, Any]: + """Return all enabled scenarios.""" + return SCENARIO_MAP + + def _execute_sql_list(backend, sql_list): """Execute a list of SQL statements.""" for sql in sql_list: @@ -154,8 +216,6 @@ def _execute_sql_list(backend, sql_list): @pytest.fixture(scope="function") def postgres_backend_single(request): """Fixture providing a single PostgreSQL backend for introspection tests.""" - from tests.providers.scenarios import get_scenario, get_enabled_scenarios - # Get first available scenario enabled_scenarios = get_enabled_scenarios() if not enabled_scenarios: @@ -173,9 +233,6 @@ def postgres_backend_single(request): @pytest_asyncio.fixture(scope="function") async def async_postgres_backend_single(request): """Async fixture providing a single PostgreSQL backend for introspection tests.""" - from tests.providers.scenarios import get_scenario, get_enabled_scenarios - from rhosocial.activerecord.backend.impl.postgres import AsyncPostgresBackend - # Get first available scenario enabled_scenarios = get_enabled_scenarios() if not enabled_scenarios: @@ -193,8 +250,6 @@ async def async_postgres_backend_single(request): @pytest.fixture(scope="function") def backend_with_tables(request): """Fixture providing backend with test tables created.""" - from tests.providers.scenarios import get_scenario, get_enabled_scenarios - # Get first available scenario enabled_scenarios = get_enabled_scenarios() if not enabled_scenarios: @@ -226,9 +281,6 @@ def backend_with_tables(request): @pytest_asyncio.fixture(scope="function") async def async_backend_with_tables(request): """Async fixture providing backend with test tables created.""" - from tests.providers.scenarios import get_scenario, get_enabled_scenarios - from rhosocial.activerecord.backend.impl.postgres import AsyncPostgresBackend - # Get first available scenario enabled_scenarios = get_enabled_scenarios() if not enabled_scenarios: @@ -263,8 +315,6 @@ async def async_backend_with_tables(request): @pytest.fixture(scope="function") def backend_with_view(request): """Fixture providing backend with test view created.""" - from tests.providers.scenarios import get_scenario, get_enabled_scenarios - enabled_scenarios = get_enabled_scenarios() if not enabled_scenarios: pytest.skip("No PostgreSQL scenarios configured") @@ -291,8 +341,6 @@ def backend_with_view(request): @pytest.fixture(scope="function") def backend_with_trigger(request): """Fixture providing backend with test trigger created.""" - from tests.providers.scenarios import get_scenario, get_enabled_scenarios - enabled_scenarios = get_enabled_scenarios() if not enabled_scenarios: pytest.skip("No PostgreSQL scenarios configured") diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_views.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_views.py index 1aa9cf4d..40ff20cf 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_views.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_views.py @@ -10,6 +10,39 @@ from rhosocial.activerecord.backend.introspection.types import ViewInfo +# SQL statements for async view tests (duplicated from conftest to avoid import issues) +_VIEW_SQL = [ + "DROP VIEW IF EXISTS user_summary CASCADE", + "DROP TABLE IF EXISTS user_stats CASCADE", + "DROP TABLE IF EXISTS users CASCADE", + """ + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + email VARCHAR(255) NOT NULL + ) + """, + """ + CREATE TABLE user_stats ( + user_id INTEGER PRIMARY KEY, + post_count INTEGER DEFAULT 0 + ) + """, + """ + CREATE VIEW user_summary AS + SELECT u.id, u.name, u.email, COALESCE(s.post_count, 0) as post_count + FROM users u + LEFT JOIN user_stats s ON u.id = s.user_id + """, +] + +_CLEANUP_VIEW_SQL = [ + "DROP VIEW IF EXISTS user_summary CASCADE", + "DROP TABLE IF EXISTS user_stats CASCADE", + "DROP TABLE IF EXISTS users CASCADE", +] + + class TestListViews: """Tests for list_views method.""" @@ -102,7 +135,6 @@ class TestAsyncViewIntrospection: async def test_async_list_views(self, async_postgres_backend_single): """Test async list_views returns view info.""" # Create view manually for async test - from tests.rhosocial.activerecord_postgres_test.feature.backend.introspection.conftest import _VIEW_SQL, _CLEANUP_VIEW_SQL for sql in _VIEW_SQL: await async_postgres_backend_single.execute(sql) async_postgres_backend_single.introspector.clear_cache() @@ -119,7 +151,6 @@ async def test_async_list_views(self, async_postgres_backend_single): @pytest.mark.asyncio async def test_async_get_view_info(self, async_postgres_backend_single): """Test async get_view_info for existing view.""" - from tests.rhosocial.activerecord_postgres_test.feature.backend.introspection.conftest import _VIEW_SQL, _CLEANUP_VIEW_SQL for sql in _VIEW_SQL: await async_postgres_backend_single.execute(sql) async_postgres_backend_single.introspector.clear_cache() From afc3c1f7c08845ada593fef7063eb723d08a5940 Mon Sep 17 00:00:00 2001 From: zhao vistart <864657+vistart@users.noreply.github.com> Date: Thu, 2 Apr 2026 14:51:37 +0800 Subject: [PATCH 03/11] feat(backend): add EXPLAIN clause support and connection recovery for PostgreSQL (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend): add explain() interface with typed PostgresExplainResult Implement backend.explain(expression, options) for PostgresBackend and AsyncPostgresBackend, following the sync/async parity principle. Changes: - New postgres/explain/ submodule with PostgresExplainPlanLine (wraps the single "QUERY PLAN" column per row) and PostgresExplainResult with analyze_index_usage() / is_full_scan / is_index_used / is_covering_index helpers (text-matching on Seq Scan / Index Scan / Index Only Scan keywords) - PostgresBackend and AsyncPostgresBackend now mixin SyncExplainBackendMixin and AsyncExplainBackendMixin respectively - Override _parse_explain_result() to extract "QUERY PLAN" key from each raw row and construct typed PostgresExplainPlanLine objects - Add format_explain_statement() to PostgresDialect using PostgreSQL bracket-option syntax: EXPLAIN [(ANALYZE, FORMAT JSON)] stmt - Export PostgresExplainResult and PostgresExplainPlanLine from the package __init__ (postgres execute() already included EXPLAIN in its DQL detection tuple — no fix needed) - Add integration tests covering protocol check, result structure, full-scan / index detection, EXPLAIN ANALYZE, and async parity * fix(dialect): return complete (sql, params) tuple from format_explain_statement The overridden format_explain_statement() in PostgresDialect was returning a plain str (the EXPLAIN prefix only), while ExplainExpression.to_sql() passes the return value directly to backend_mixin which unpacks it as (sql, params). This caused a ValueError at runtime. Fix by building the full EXPLAIN SQL string inside the method and returning (sql, params) consistent with the base class contract. Also fix test_backend_explain.py: replace executescript() calls (a MySQL-only method) with per-statement execute() calls, matching the pattern used in all other PostgreSQL backend tests. Add EXPLAIN documentation (en_US and zh_CN) and update the feature index README files. * feat(backend): add dual-layer connection recovery for PostgreSQL Implement automatic reconnection mechanism with two layers: - Plan A: Pre-query connection check in _get_cursor() using psycopg's closed/broken attributes - Plan B: Error retry mechanism in execute() detecting connection errors via SQLSTATE codes and error message patterns Supported connection error SQLSTATE codes: - 08000-08007: Connection exception class - 57P01-57P04: Server shutdown class Added fixtures for connection resilience testing and comprehensive documentation covering connection loss scenarios and recovery behavior. * ci: update rhosocial-activerecord installation source to git+https Install from release/v1.0.0.dev21 branch instead of PyPI dev20. * chore: add changelog fragment for PR #16 --- .github/workflows/test.yml | 2 +- changelog.d/16.added.md | 1 + .../postgres_specific_features/README.md | 1 + .../postgres_specific_features/explain.md | 294 +++++++++++++++ docs/en_US/troubleshooting/connection.md | 208 ++++++++++- .../postgres_specific_features/README.md | 1 + .../postgres_specific_features/explain.md | 294 +++++++++++++++ docs/zh_CN/troubleshooting/connection.md | 208 ++++++++++- .../backend/impl/postgres/__init__.py | 4 + .../impl/postgres/backend/async_backend.py | 240 +++++++++++-- .../backend/impl/postgres/backend/sync.py | 240 +++++++++++-- .../backend/impl/postgres/dialect.py | 34 ++ .../backend/impl/postgres/explain/__init__.py | 10 + .../backend/impl/postgres/explain/types.py | 97 +++++ .../feature/backend/conftest.py | 76 ++++ .../feature/backend/test_backend_explain.py | 282 +++++++++++++++ .../backend/test_connection_resilience.py | 340 ++++++++++++++++++ 17 files changed, 2259 insertions(+), 73 deletions(-) create mode 100644 changelog.d/16.added.md create mode 100644 docs/en_US/postgres_specific_features/explain.md create mode 100644 docs/zh_CN/postgres_specific_features/explain.md create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/explain/__init__.py create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/explain/types.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_backend_explain.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_connection_resilience.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3b23dde..a06c5b22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: - name: Install activerecord and testsuite from PyPI run: | - pip install rhosocial-activerecord==1.0.0.dev20 + pip install git+https://github.com/rhosocial/python-activerecord@release/v1.0.0.dev21 pip install rhosocial-activerecord-testsuite==1.0.0.dev7 - name: Create postgres scenario config diff --git a/changelog.d/16.added.md b/changelog.d/16.added.md new file mode 100644 index 00000000..8e871dad --- /dev/null +++ b/changelog.d/16.added.md @@ -0,0 +1 @@ +Added EXPLAIN clause support with typed PostgresExplainResult and dual-layer connection recovery for PostgreSQL backend diff --git a/docs/en_US/postgres_specific_features/README.md b/docs/en_US/postgres_specific_features/README.md index 068300cf..57232011 100644 --- a/docs/en_US/postgres_specific_features/README.md +++ b/docs/en_US/postgres_specific_features/README.md @@ -10,6 +10,7 @@ PostgreSQL offers many advanced features beyond standard SQL. This section cover - **[RETURNING Clause](./returning.md)**: Get data back from DML operations - **[Protocol Support Matrix](./protocol_support.md)**: Comprehensive protocol support and version compatibility - **[Database Introspection](./introspection.md)**: Query metadata using pg_catalog +- **[EXPLAIN Support](./explain.md)**: Query execution plan analysis and performance diagnostics ## Feature Highlights diff --git a/docs/en_US/postgres_specific_features/explain.md b/docs/en_US/postgres_specific_features/explain.md new file mode 100644 index 00000000..2f633ea8 --- /dev/null +++ b/docs/en_US/postgres_specific_features/explain.md @@ -0,0 +1,294 @@ +# EXPLAIN Support + +## Overview + +The PostgreSQL backend provides comprehensive support for the `EXPLAIN` statement, enabling query performance analysis and execution plan inspection. This feature integrates with the ActiveRecord query builder through a unified API while exposing PostgreSQL-specific behaviors. + +Key capabilities: +- Basic `EXPLAIN` for execution plan visualization +- `EXPLAIN ANALYZE` for actual runtime statistics (all modern versions) +- Multiple output formats: `TEXT`, `XML`, `JSON`, `YAML` +- PostgreSQL's bracketed option syntax (`EXPLAIN (ANALYZE, FORMAT JSON)`) +- Structured result parsing with `PostgresExplainResult` +- Index usage analysis helpers (`Seq Scan`, `Index Scan`, `Index Only Scan`) +- Full sync and async support + +## Basic Usage + +### Accessing EXPLAIN via Query Builder + +```python +from rhosocial.activerecord.backend.expression.statements import ExplainFormat + +# Simple EXPLAIN — returns PostgresExplainResult +result = User.query().explain().all() + +# Inspect the generated SQL prefix +print(result.sql) # EXPLAIN SELECT ... + +# Print the execution time (seconds) +print(result.duration) +``` + +### Reading the Result Rows + +```python +result = User.query().explain().all() + +for row in result.rows: + print(row.line) +``` + +Each row is a `PostgresExplainPlanLine` with a single field: + +| Field | Type | Description | +|---|---|---| +| `line` | `str` | One line of the query plan text (e.g. `"Seq Scan on users (cost=0.00..8.27 rows=1 width=...)"`). | + +PostgreSQL returns each plan node as an indented text line in the `"QUERY PLAN"` column. All lines together form the full execution plan tree. + +## Output Formats + +PostgreSQL supports four output formats, all available on any modern server version: + +### TEXT Format (Default) + +```python +# Default — no FORMAT option emitted +result = User.query().explain().all() +# Generates: EXPLAIN SELECT ... +``` + +### JSON Format + +```python +result = User.query().explain(format=ExplainFormat.JSON).all() +# Generates: EXPLAIN (FORMAT JSON) SELECT ... +``` + +### XML Format + +```python +result = User.query().explain(format=ExplainFormat.XML).all() +# Generates: EXPLAIN (FORMAT XML) SELECT ... +``` + +### YAML Format + +```python +result = User.query().explain(format=ExplainFormat.YAML).all() +# Generates: EXPLAIN (FORMAT YAML) SELECT ... +``` + +## EXPLAIN ANALYZE + +`EXPLAIN ANALYZE` executes the query and returns actual runtime statistics alongside cost estimates. Use it to compare estimated vs. actual row counts and timing. + +```python +# Basic ANALYZE +result = User.query().explain(analyze=True).all() +# Generates: EXPLAIN (ANALYZE) SELECT ... + +# ANALYZE with JSON format +result = User.query().explain(analyze=True, format=ExplainFormat.JSON).all() +# Generates: EXPLAIN (ANALYZE, FORMAT JSON) SELECT ... + +# ANALYZE with YAML format +result = User.query().explain(analyze=True, format=ExplainFormat.YAML).all() +# Generates: EXPLAIN (ANALYZE, FORMAT YAML) SELECT ... +``` + +**Important Notes:** +- `EXPLAIN ANALYZE` **executes the query** against the database. Avoid using it on write statements in production without understanding the side effects. +- Supported on all modern PostgreSQL versions (no minimum version constraint within the supported range). + +## SQL Syntax Generated + +PostgreSQL uses a parenthesized option syntax, distinct from MySQL's flat syntax: + +``` +EXPLAIN [ ( option [, ...] ) ] statement +``` + +Options are comma-separated inside parentheses: + +```python +# No options +# → EXPLAIN SELECT ... + +# analyze only +# → EXPLAIN (ANALYZE) SELECT ... + +# format only +# → EXPLAIN (FORMAT JSON) SELECT ... + +# both analyze and format +# → EXPLAIN (ANALYZE, FORMAT JSON) SELECT ... +``` + +The `ANALYZE` option always appears before `FORMAT` in the option list. + +## Index Usage Analysis + +`PostgresExplainResult` provides helpers for a quick index usage assessment based on text matching across all plan lines: + +```python +result = User.query().where(User.c.email == "alice@example.com").explain().all() + +usage = result.analyze_index_usage() +print(usage) # "full_scan" | "index_with_lookup" | "covering_index" | "unknown" + +# Convenience properties +if result.is_full_scan: + print("Warning: sequential scan detected") + +if result.is_covering_index: + print("Optimal: index only scan in use") + +if result.is_index_used: + print("An index is being used") +``` + +### Index Usage Rules + +The analysis concatenates all plan lines and performs case-insensitive text matching: + +| Matched keyword (case-insensitive) | `analyze_index_usage()` return | +|---|---| +| `"INDEX ONLY SCAN"` (checked first) | `"covering_index"` | +| `"INDEX SCAN"`, `"BITMAP INDEX SCAN"`, or `"BITMAP HEAP SCAN"` | `"index_with_lookup"` | +| `"SEQ SCAN"` | `"full_scan"` | +| None of the above | `"unknown"` | + +**Note:** `"INDEX ONLY SCAN"` is checked before `"INDEX SCAN"` to avoid misclassification. + +## Async API + +The async backend provides the same interface. Use `await` for the terminal method: + +```python +async def analyze_queries(): + # Simple async EXPLAIN + result = await User.query().explain().all_async() + + # ANALYZE with JSON format + result = await User.query().explain( + analyze=True, + format=ExplainFormat.JSON + ).all_async() + + for row in result.rows: + print(row.line) +``` + +## Checking Format Support at Runtime + +Use `dialect` methods to guard option availability: + +```python +dialect = backend.dialect + +# Check ANALYZE support (always True for PostgreSQL) +if dialect.supports_explain_analyze(): + result = User.query().explain(analyze=True).all() + +# Check a specific format (all four formats always supported) +if dialect.supports_explain_format("JSON"): + result = User.query().explain(format=ExplainFormat.JSON).all() +``` + +## Combining EXPLAIN with Complex Queries + +EXPLAIN works with the full query builder chain: + +```python +# JOIN query +result = ( + User.query() + .join(Order, User.c.id == Order.c.user_id) + .where(User.c.status == "active") + .explain(format=ExplainFormat.JSON) + .all() +) + +# GROUP BY / aggregate +result = ( + Order.query() + .group_by(Order.c.user_id) + .explain(analyze=True) + .count(Order.c.id) +) + +# Subquery +result = ( + User.query() + .where(User.c.id.in_(Order.query().select(Order.c.user_id))) + .explain() + .all() +) +``` + +## Comparison with MySQL EXPLAIN + +Both backends share the same query builder API but differ in SQL syntax and result structure: + +| Feature | PostgreSQL | MySQL | +|---|---|---| +| SQL syntax | `EXPLAIN (ANALYZE, FORMAT JSON)` | `EXPLAIN ANALYZE FORMAT=JSON` | +| Option delimiter | Comma-separated, parenthesized | Space-separated, no parentheses | +| Supported formats | TEXT, XML, JSON, YAML | TEXT, JSON (5.6.5+), TREE (8.0.16+) | +| ANALYZE support | All modern versions | 8.0.18+ | +| Result row type | `PostgresExplainPlanLine` (1 field) | `MySQLExplainRow` (12 fields) | +| Covering index indicator | `"Index Only Scan"` in plan text | `"Using index"` in `extra` column | + +## Version Compatibility + +| Feature | Minimum PostgreSQL Version | +|---|---| +| Basic `EXPLAIN` | All supported versions | +| `EXPLAIN FORMAT=TEXT/XML/JSON/YAML` | All supported versions | +| `EXPLAIN ANALYZE` | All supported versions | +| `EXPLAIN (ANALYZE, FORMAT JSON)` | All supported versions | + +The PostgreSQL backend targets PostgreSQL 9.6 and later; all EXPLAIN features described here are available on 9.6+. + +## API Reference + +### `ExplainOptions` Fields Used by PostgreSQL + +| Field | Type | Default | Description | +|---|---|---|---| +| `analyze` | `bool` | `False` | Emit `ANALYZE` option | +| `format` | `Optional[ExplainFormat]` | `None` | Output format (`TEXT`/`XML`/`JSON`/`YAML`) | + +Other `ExplainOptions` fields (such as `verbose`, `buffers`, `timing`, `settings`) are accepted but not yet emitted by the current PostgreSQL dialect implementation. + +### `PostgresExplainResult` Methods + +| Method / Property | Description | +|---|---| +| `rows` | `List[PostgresExplainPlanLine]` — parsed plan lines | +| `sql` | The full SQL string that was explained | +| `duration` | Query execution time in seconds | +| `raw_rows` | Raw rows as returned by psycopg (list of dicts with `"QUERY PLAN"` key) | +| `analyze_index_usage()` | Returns `"full_scan"`, `"index_with_lookup"`, `"covering_index"`, or `"unknown"` | +| `is_full_scan` | `True` if `analyze_index_usage() == "full_scan"` | +| `is_index_used` | `True` if any index is used | +| `is_covering_index` | `True` if an index-only scan is used | + +### `PostgresDialect` Methods + +| Method | Return | Description | +|---|---|---| +| `supports_explain_analyze()` | `bool` | Always `True` for PostgreSQL | +| `supports_explain_format(fmt)` | `bool` | `True` for `TEXT`, `XML`, `JSON`, `YAML` | +| `format_explain_statement(expr)` | `str` | Builds `EXPLAIN [(ANALYZE, FORMAT X)]` prefix | + +## Best Practices + +- **Use JSON format for programmatic analysis.** `FORMAT JSON` produces a structured plan tree that is easier to parse than plain text and provides the most detail. +- **Reserve `EXPLAIN ANALYZE` for development.** The statement executes the query, so use it only when you need actual row counts and timing figures. +- **Leverage all four formats for different audiences.** Use `TEXT` for human inspection, `JSON`/`YAML` for tooling, and `XML` for XML-based reporting pipelines. +- **Combine with index analysis.** Use `result.is_full_scan` as a quick smoke test in integration tests to detect inadvertent sequential scans. + +💡 *AI Prompt: Explain how to use EXPLAIN ANALYZE with JSON format on PostgreSQL to diagnose slow queries in rhosocial-activerecord.* diff --git a/docs/en_US/troubleshooting/connection.md b/docs/en_US/troubleshooting/connection.md index a3228306..7de866e0 100644 --- a/docs/en_US/troubleshooting/connection.md +++ b/docs/en_US/troubleshooting/connection.md @@ -105,4 +105,210 @@ psql -U postgres -c "CREATE DATABASE test;" createdb -U postgres test ``` -💡 *AI Prompt:* "How to troubleshoot PostgreSQL connection errors?" +## Connection Loss and Automatic Recovery + +### Overview + +In long-running applications, database connections may be dropped for various reasons. The PostgreSQL backend implements a dual-layer protection mechanism to ensure automatic recovery when connections are lost. + +### Common Connection Loss Scenarios + +| Scenario | Cause | SQLSTATE Codes | +|----------|-------|----------------| +| `idle_in_transaction_session_timeout` expiry | Transaction idle time exceeds PostgreSQL's setting | 08006 | +| Connection terminated | DBA executes `pg_terminate_backend()` | 57P01 | +| Network instability | Network issues cause TCP connection to drop | 08006 | +| Server restart | PostgreSQL server restart or crash | 08006, 57P01 | +| Firewall timeout | Firewall closes long-idle TCP connections | 08006 | +| Admin shutdown | Server is being shut down by administrator | 57P01, 57P02 | + +### Automatic Recovery Mechanism + +The PostgreSQL backend implements two layers of automatic recovery: + +#### Plan A: Pre-Query Connection Check + +Before each query execution, the backend automatically checks the connection status: + +```python +def _get_cursor(self): + """Get a database cursor, ensuring connection is active.""" + if not self._connection: + # No connection, establish new one + self.connect() + elif self._connection.closed or self._connection.broken: + # Connection lost, reconnect + self._connection = None + self.connect() + return self._connection.cursor() +``` + +**Features**: +- Proactive checking, detects issues before query execution +- Uses psycopg v3's `closed` and `broken` attributes to check connection state +- Completely transparent to the application layer + +**Note**: Unlike MySQL's `is_connected()` method which actively polls the server, psycopg v3's `closed` and `broken` attributes are updated lazily. They may not immediately reflect server-side disconnection until the next operation is attempted. This is why Plan B (error retry) is also necessary. + +#### Plan B: Error Retry Mechanism + +When a connection error occurs during query execution, the backend automatically retries: + +```python +# PostgreSQL connection error SQLSTATE codes +CONNECTION_ERROR_SQLSTATES = { + "08000", # CONNECTION_EXCEPTION + "08001", # SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION + "08003", # CONNECTION_DOES_NOT_EXIST + "08004", # SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION + "08006", # CONNECTION_FAILURE + "08007", # TRANSACTION_RESOLUTION_UNKNOWN + "08P01", # PROTOCOL_VIOLATION + "57P01", # ADMIN_SHUTDOWN + "57P02", # CRASH_SHUTDOWN + "57P03", # CANNOT_CONNECT_NOW + "57P04", # DATABASE_DROPPED +} + +async def execute(self, sql, params=None, *, options=None, max_retries=2): + for attempt in range(max_retries + 1): + try: + return await super().execute(sql, params, options=options) + except OperationalError as e: + if self._is_connection_error_message(str(e)) and attempt < max_retries: + await self._reconnect() + continue + raise +``` + +**Features**: +- Reactive recovery, triggered on query failure +- Maximum 2 retries +- Only retries on connection errors, other errors are raised directly +- Detects connection errors via SQLSTATE codes and error message patterns + +### Manual Keep-Alive Mechanism + +For scenarios requiring proactive connection maintenance, use the `ping()` method: + +```python +# Check connection status without auto-reconnect +is_alive = backend.ping(reconnect=False) + +# Check connection status with auto-reconnect if disconnected +is_alive = backend.ping(reconnect=True) +``` + +#### Keep-Alive Best Practices + +In multi-process worker scenarios, implement periodic keep-alive: + +```python +import threading +import time + +def keepalive_worker(backend, interval=60): + """Background keep-alive thread""" + while True: + time.sleep(interval) + if backend.ping(reconnect=True): + logger.debug("Connection keepalive successful") + else: + logger.warning("Connection keepalive failed") + +# Start keep-alive thread +keepalive_thread = threading.Thread( + target=keepalive_worker, + args=(backend, 60), + daemon=True +) +keepalive_thread.start() +``` + +### Async Backend Support + +The async backend (`AsyncPostgresBackend`) provides the same connection recovery mechanism: + +```python +# Async ping +is_alive = await async_backend.ping(reconnect=True) + +# Async queries also auto-reconnect +result = await async_backend.execute("SELECT 1") +``` + +### Best Practices + +#### 1. Configure PostgreSQL Timeout Parameters Appropriately + +```sql +-- View current settings +SHOW idle_in_transaction_session_timeout; +SHOW statement_timeout; + +-- Recommended settings (adjust based on your requirements) +SET GLOBAL idle_in_transaction_session_timeout = '10min'; +-- statement_timeout should be set per-session based on query complexity +``` + +#### 2. Use Connection Pool + +For high-concurrency scenarios, configure connection pooling: + +```python +config = PostgreSQLConnectionConfig( + host='localhost', + database='mydb', + # Connection pool settings + # Note: psycopg v3 has built-in connection pool support +) +``` + +#### 3. Multi-Process Worker Scenarios + +Each worker process should have its own backend instance: + +```python +def worker_process(worker_id, config): + """Worker process entry point""" + # Create independent backend instance within the process + backend = PostgresBackend(connection_config=config) + backend.connect() + + try: + # Execute tasks + do_work(backend) + finally: + backend.disconnect() +``` + +#### 4. Monitor Connection Status + +```python +import logging + +# Enable backend logging +logging.getLogger('rhosocial.activerecord.backend').setLevel(logging.DEBUG) + +# Backend will automatically log connection recovery events +# DEBUG: Connection lost, reconnecting... +# DEBUG: Reconnected successfully +``` + +### Connection Error SQLSTATE Codes Reference + +| SQLSTATE | Name | Description | +|----------|------|-------------| +| 08000 | CONNECTION_EXCEPTION | General connection exception | +| 08001 | SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION | Client unable to establish connection | +| 08003 | CONNECTION_DOES_NOT_EXIST | Connection does not exist | +| 08004 | SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION | Server rejected connection | +| 08006 | CONNECTION_FAILURE | Connection failure | +| 08007 | TRANSACTION_RESOLUTION_UNKNOWN | Transaction resolution unknown | +| 08P01 | PROTOCOL_VIOLATION | Protocol violation | +| 57P01 | ADMIN_SHUTDOWN | Administrator commanded shutdown | +| 57P02 | CRASH_SHUTDOWN | Server crash | +| 57P03 | CANNOT_CONNECT_NOW | Cannot connect now | +| 57P04 | DATABASE_DROPPED | Database dropped | + +💡 *AI Prompt:* "How does PostgreSQL backend automatically recover from connection losses? What is the dual-layer protection mechanism?" diff --git a/docs/zh_CN/postgres_specific_features/README.md b/docs/zh_CN/postgres_specific_features/README.md index 510b0db8..8d7d73ab 100644 --- a/docs/zh_CN/postgres_specific_features/README.md +++ b/docs/zh_CN/postgres_specific_features/README.md @@ -10,6 +10,7 @@ PostgreSQL 提供了许多超越标准 SQL 的高级功能。本节介绍 Postgr - **[RETURNING 子句](./returning.md)**: 从 DML 操作获取返回数据 - **[协议支持矩阵](./protocol_support.md)**: 完整的协议支持和版本兼容性 - **[数据库内省](./introspection.md)**: 使用 pg_catalog 查询元数据 +- **[EXPLAIN 支持](./explain.md)**: 查询执行计划分析与性能诊断 ## 功能亮点 diff --git a/docs/zh_CN/postgres_specific_features/explain.md b/docs/zh_CN/postgres_specific_features/explain.md new file mode 100644 index 00000000..db1bde31 --- /dev/null +++ b/docs/zh_CN/postgres_specific_features/explain.md @@ -0,0 +1,294 @@ +# EXPLAIN 支持 + +## 概述 + +PostgreSQL 后端提供对 `EXPLAIN` 语句的完整支持，可用于分析查询性能并查看执行计划。该功能通过统一的 API 与 ActiveRecord 查询构建器集成，同时暴露 PostgreSQL 特有的行为。 + +主要能力： +- 基础 `EXPLAIN`，用于查看执行计划 +- `EXPLAIN ANALYZE`，返回实际运行时统计数据（所有现代版本均支持） +- 多种输出格式：`TEXT`、`XML`、`JSON`、`YAML` +- PostgreSQL 括号式选项语法（`EXPLAIN (ANALYZE, FORMAT JSON)`） +- 结构化结果解析，通过 `PostgresExplainResult` 访问 +- 索引使用分析辅助方法（`Seq Scan`、`Index Scan`、`Index Only Scan`） +- 同步与异步双模式支持 + +## 基本用法 + +### 通过查询构建器使用 EXPLAIN + +```python +from rhosocial.activerecord.backend.expression.statements import ExplainFormat + +# 简单 EXPLAIN — 返回 PostgresExplainResult +result = User.query().explain().all() + +# 查看生成的 SQL 前缀 +print(result.sql) # EXPLAIN SELECT ... + +# 打印执行耗时（秒） +print(result.duration) +``` + +### 读取结果行 + +```python +result = User.query().explain().all() + +for row in result.rows: + print(row.line) +``` + +每行是一个 `PostgresExplainPlanLine`，只有一个字段： + +| 字段 | 类型 | 说明 | +|---|---|---| +| `line` | `str` | 一行计划文本（如 `"Seq Scan on users (cost=0.00..8.27 rows=1 width=...)"`） | + +PostgreSQL 将每个计划节点以缩进文本行的形式写入 `"QUERY PLAN"` 列。所有行合在一起构成完整的执行计划树。 + +## 输出格式 + +PostgreSQL 支持四种输出格式，所有现代服务器版本均可使用： + +### TEXT 格式（默认） + +```python +# 默认情况下不生成 FORMAT 选项 +result = User.query().explain().all() +# 生成: EXPLAIN SELECT ... +``` + +### JSON 格式 + +```python +result = User.query().explain(format=ExplainFormat.JSON).all() +# 生成: EXPLAIN (FORMAT JSON) SELECT ... +``` + +### XML 格式 + +```python +result = User.query().explain(format=ExplainFormat.XML).all() +# 生成: EXPLAIN (FORMAT XML) SELECT ... +``` + +### YAML 格式 + +```python +result = User.query().explain(format=ExplainFormat.YAML).all() +# 生成: EXPLAIN (FORMAT YAML) SELECT ... +``` + +## EXPLAIN ANALYZE + +`EXPLAIN ANALYZE` 会实际执行查询，并在代价估算的基础上同时返回真实的运行时统计数据，可用于对比预估行数与实际行数及耗时。 + +```python +# 基础 ANALYZE +result = User.query().explain(analyze=True).all() +# 生成: EXPLAIN (ANALYZE) SELECT ... + +# 带 JSON 格式的 ANALYZE +result = User.query().explain(analyze=True, format=ExplainFormat.JSON).all() +# 生成: EXPLAIN (ANALYZE, FORMAT JSON) SELECT ... + +# 带 YAML 格式的 ANALYZE +result = User.query().explain(analyze=True, format=ExplainFormat.YAML).all() +# 生成: EXPLAIN (ANALYZE, FORMAT YAML) SELECT ... +``` + +**重要说明：** +- `EXPLAIN ANALYZE` **会真正执行查询**。在生产环境中对写操作使用时请谨慎评估副作用。 +- 所有现代 PostgreSQL 版本均支持，在支持的版本范围内无最低版本限制。 + +## 生成的 SQL 语法 + +PostgreSQL 使用括号式选项语法，与 MySQL 的平铺式语法不同： + +``` +EXPLAIN [ ( option [, ...] ) ] 语句 +``` + +多个选项在括号内以逗号分隔： + +```python +# 无选项 +# → EXPLAIN SELECT ... + +# 仅 analyze=True +# → EXPLAIN (ANALYZE) SELECT ... + +# 仅指定 format +# → EXPLAIN (FORMAT JSON) SELECT ... + +# 同时指定两者 +# → EXPLAIN (ANALYZE, FORMAT JSON) SELECT ... +``` + +`ANALYZE` 选项始终在选项列表中排在 `FORMAT` 之前。 + +## 索引使用分析 + +`PostgresExplainResult` 提供基于文本匹配的索引使用快速评估，通过对所有计划行进行关键词搜索： + +```python +result = User.query().where(User.c.email == "alice@example.com").explain().all() + +usage = result.analyze_index_usage() +print(usage) # "full_scan" | "index_with_lookup" | "covering_index" | "unknown" + +# 便捷属性 +if result.is_full_scan: + print("警告：检测到顺序扫描") + +if result.is_covering_index: + print("最优：正在使用 Index Only Scan") + +if result.is_index_used: + print("索引已被使用") +``` + +### 索引使用判断规则 + +分析将所有计划行拼接后进行大小写不敏感的文本匹配： + +| 匹配关键词（不区分大小写） | `analyze_index_usage()` 返回值 | +|---|---| +| `"INDEX ONLY SCAN"`（优先检查） | `"covering_index"` | +| `"INDEX SCAN"`、`"BITMAP INDEX SCAN"` 或 `"BITMAP HEAP SCAN"` | `"index_with_lookup"` | +| `"SEQ SCAN"` | `"full_scan"` | +| 以上均不匹配 | `"unknown"` | + +**注意：** `"INDEX ONLY SCAN"` 优先于 `"INDEX SCAN"` 检查，以避免误判。 + +## 异步 API + +异步后端提供相同接口，在终止方法处使用 `await`： + +```python +async def analyze_queries(): + # 简单异步 EXPLAIN + result = await User.query().explain().all_async() + + # 带 JSON 格式的 ANALYZE + result = await User.query().explain( + analyze=True, + format=ExplainFormat.JSON + ).all_async() + + for row in result.rows: + print(row.line) +``` + +## 运行时检查格式支持 + +使用 `dialect` 方法在调用前检查选项可用性： + +```python +dialect = backend.dialect + +# 检查 ANALYZE 支持（PostgreSQL 始终为 True） +if dialect.supports_explain_analyze(): + result = User.query().explain(analyze=True).all() + +# 检查特定格式（四种格式始终支持） +if dialect.supports_explain_format("JSON"): + result = User.query().explain(format=ExplainFormat.JSON).all() +``` + +## 与复杂查询组合使用 + +EXPLAIN 可与完整的查询构建器链式调用配合： + +```python +# JOIN 查询 +result = ( + User.query() + .join(Order, User.c.id == Order.c.user_id) + .where(User.c.status == "active") + .explain(format=ExplainFormat.JSON) + .all() +) + +# GROUP BY / 聚合 +result = ( + Order.query() + .group_by(Order.c.user_id) + .explain(analyze=True) + .count(Order.c.id) +) + +# 子查询 +result = ( + User.query() + .where(User.c.id.in_(Order.query().select(Order.c.user_id))) + .explain() + .all() +) +``` + +## 与 MySQL EXPLAIN 的对比 + +两个后端共享相同的查询构建器 API，但在 SQL 语法和结果结构上存在差异： + +| 特性 | PostgreSQL | MySQL | +|---|---|---| +| SQL 语法 | `EXPLAIN (ANALYZE, FORMAT JSON)` | `EXPLAIN ANALYZE FORMAT=JSON` | +| 选项分隔符 | 括号内逗号分隔 | 空格分隔，无括号 | +| 支持的格式 | TEXT、XML、JSON、YAML | TEXT、JSON（5.6.5+）、TREE（8.0.16+） | +| ANALYZE 支持 | 所有现代版本 | 8.0.18+ | +| 结果行类型 | `PostgresExplainPlanLine`（1 个字段） | `MySQLExplainRow`（12 个字段） | +| 覆盖索引标志 | 计划文本中含 `"Index Only Scan"` | `extra` 列含 `"Using index"` | + +## 版本兼容性 + +| 功能 | 最低 PostgreSQL 版本 | +|---|---| +| 基础 `EXPLAIN` | 所有支持版本 | +| `EXPLAIN FORMAT=TEXT/XML/JSON/YAML` | 所有支持版本 | +| `EXPLAIN ANALYZE` | 所有支持版本 | +| `EXPLAIN (ANALYZE, FORMAT JSON)` | 所有支持版本 | + +PostgreSQL 后端面向 PostgreSQL 9.6 及更高版本；本文档描述的所有 EXPLAIN 功能在 9.6+ 均可使用。 + +## API 参考 + +### PostgreSQL 使用的 `ExplainOptions` 字段 + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `analyze` | `bool` | `False` | 生成 `ANALYZE` 选项 | +| `format` | `Optional[ExplainFormat]` | `None` | 输出格式（`TEXT`/`XML`/`JSON`/`YAML`） | + +其他 `ExplainOptions` 字段（如 `verbose`、`buffers`、`timing`、`settings`）会被接受，但当前 PostgreSQL 方言实现尚未将其写入生成的 SQL。 + +### `PostgresExplainResult` 方法 + +| 方法 / 属性 | 说明 | +|---|---| +| `rows` | `List[PostgresExplainPlanLine]`，解析后的计划行 | +| `sql` | 被 EXPLAIN 的完整 SQL 字符串 | +| `duration` | 查询执行耗时（秒） | +| `raw_rows` | psycopg 返回的原始行数据（包含 `"QUERY PLAN"` 键的字典列表） | +| `analyze_index_usage()` | 返回 `"full_scan"`、`"index_with_lookup"`、`"covering_index"` 或 `"unknown"` | +| `is_full_scan` | `analyze_index_usage() == "full_scan"` 时为 `True` | +| `is_index_used` | 有任意索引被使用时为 `True` | +| `is_covering_index` | 使用了 Index Only Scan 时为 `True` | + +### `PostgresDialect` 方法 + +| 方法 | 返回值 | 说明 | +|---|---|---| +| `supports_explain_analyze()` | `bool` | PostgreSQL 始终为 `True` | +| `supports_explain_format(fmt)` | `bool` | `TEXT`、`XML`、`JSON`、`YAML` 均为 `True` | +| `format_explain_statement(expr)` | `str` | 构建 `EXPLAIN [(ANALYZE, FORMAT X)]` 前缀 | + +## 最佳实践 + +- **程序化分析推荐使用 JSON 格式。** `FORMAT JSON` 提供最为详细的结构化计划树，比纯文本更易解析。 +- **`EXPLAIN ANALYZE` 仅用于开发阶段。** 该语句会真正执行查询，仅在需要实际行数和耗时数据时使用。 +- **针对不同受众选择不同格式。** 人工查阅使用 `TEXT`，工具集成使用 `JSON`/`YAML`，XML 报表管道使用 `XML`。 +- **结合索引分析。** 在集成测试中使用 `result.is_full_scan` 作为快速冒烟测试，及时发现意外的顺序扫描。 + +💡 *AI 提示词：说明如何在 rhosocial-activerecord 中使用 PostgreSQL 的 EXPLAIN ANALYZE FORMAT JSON 诊断慢查询。* diff --git a/docs/zh_CN/troubleshooting/connection.md b/docs/zh_CN/troubleshooting/connection.md index a483f3c1..64dc25c3 100644 --- a/docs/zh_CN/troubleshooting/connection.md +++ b/docs/zh_CN/troubleshooting/connection.md @@ -105,4 +105,210 @@ psql -U postgres -c "CREATE DATABASE test;" createdb -U postgres test ``` -💡 *AI 提示词：* "如何排查 PostgreSQL 连接错误？" +## 连接丢失与自动恢复 + +### 概述 + +在长时间运行的应用程序中，数据库连接可能因各种原因被断开。PostgreSQL 后端实现了双层保护机制，确保连接丢失时能够自动恢复。 + +### 常见连接丢失场景 + +| 场景 | 原因 | SQLSTATE 代码 | +|------|------|---------------| +| `idle_in_transaction_session_timeout` 超时 | 事务空闲时间超过 PostgreSQL 设置 | 08006 | +| 连接被终止 | DBA 执行 `pg_terminate_backend()` | 57P01 | +| 网络不稳定 | 网络问题导致 TCP 连接断开 | 08006 | +| 服务器重启 | PostgreSQL 服务器重启或崩溃 | 08006, 57P01 | +| 防火墙超时 | 防火墙关闭长时间空闲的 TCP 连接 | 08006 | +| 管理员关闭 | 服务器正在被管理员关闭 | 57P01, 57P02 | + +### 自动恢复机制 + +PostgreSQL 后端实现了两层自动恢复： + +#### 方案 A：查询前连接检查 + +在每次查询执行前，后端自动检查连接状态： + +```python +def _get_cursor(self): + """获取数据库游标，确保连接处于活动状态""" + if not self._connection: + # 无连接，建立新连接 + self.connect() + elif self._connection.closed or self._connection.broken: + # 连接已断开，重新连接 + self._connection = None + self.connect() + return self._connection.cursor() +``` + +**特点**： +- 主动检查，在查询执行前检测问题 +- 使用 psycopg v3 的 `closed` 和 `broken` 属性检查连接状态 +- 对应用层完全透明 + +**注意**：与 MySQL 的 `is_connected()` 方法主动轮询服务器不同，psycopg v3 的 `closed` 和 `broken` 属性是惰性更新的。它们可能不会立即反映服务器端的断开，直到尝试下一次操作。这就是为什么方案 B（错误重试）也是必要的。 + +#### 方案 B：错误重试机制 + +当查询执行过程中发生连接错误时，后端自动重试： + +```python +# PostgreSQL 连接错误 SQLSTATE 代码 +CONNECTION_ERROR_SQLSTATES = { + "08000", # CONNECTION_EXCEPTION + "08001", # SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION + "08003", # CONNECTION_DOES_NOT_EXIST + "08004", # SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION + "08006", # CONNECTION_FAILURE + "08007", # TRANSACTION_RESOLUTION_UNKNOWN + "08P01", # PROTOCOL_VIOLATION + "57P01", # ADMIN_SHUTDOWN + "57P02", # CRASH_SHUTDOWN + "57P03", # CANNOT_CONNECT_NOW + "57P04", # DATABASE_DROPPED +} + +async def execute(self, sql, params=None, *, options=None, max_retries=2): + for attempt in range(max_retries + 1): + try: + return await super().execute(sql, params, options=options) + except OperationalError as e: + if self._is_connection_error_message(str(e)) and attempt < max_retries: + await self._reconnect() + continue + raise +``` + +**特点**： +- 被动恢复，在查询失败时触发 +- 最多重试 2 次 +- 仅对连接错误重试，其他错误直接抛出 +- 通过 SQLSTATE 代码和错误消息模式检测连接错误 + +### 手动保活机制 + +对于需要主动维护连接的场景，使用 `ping()` 方法： + +```python +# 检查连接状态，不自动重连 +is_alive = backend.ping(reconnect=False) + +# 检查连接状态，如果断开则自动重连 +is_alive = backend.ping(reconnect=True) +``` + +#### 保活最佳实践 + +在多进程 Worker 场景中，实现定期保活： + +```python +import threading +import time + +def keepalive_worker(backend, interval=60): + """后台保活线程""" + while True: + time.sleep(interval) + if backend.ping(reconnect=True): + logger.debug("连接保活成功") + else: + logger.warning("连接保活失败") + +# 启动保活线程 +keepalive_thread = threading.Thread( + target=keepalive_worker, + args=(backend, 60), + daemon=True +) +keepalive_thread.start() +``` + +### 异步后端支持 + +异步后端（`AsyncPostgresBackend`）提供相同的连接恢复机制： + +```python +# 异步 ping +is_alive = await async_backend.ping(reconnect=True) + +# 异步查询也会自动重连 +result = await async_backend.execute("SELECT 1") +``` + +### 最佳实践 + +#### 1. 合理配置 PostgreSQL 超时参数 + +```sql +-- 查看当前设置 +SHOW idle_in_transaction_session_timeout; +SHOW statement_timeout; + +-- 推荐设置（根据您的需求调整） +SET GLOBAL idle_in_transaction_session_timeout = '10min'; +-- statement_timeout 应根据查询复杂度在每个会话中设置 +``` + +#### 2. 使用连接池 + +对于高并发场景，配置连接池： + +```python +config = PostgreSQLConnectionConfig( + host='localhost', + database='mydb', + # 连接池设置 + # 注意：psycopg v3 内置连接池支持 +) +``` + +#### 3. 多进程 Worker 场景 + +每个 Worker 进程应有自己的后端实例： + +```python +def worker_process(worker_id, config): + """Worker 进程入口""" + # 在进程内创建独立的后端实例 + backend = PostgresBackend(connection_config=config) + backend.connect() + + try: + # 执行任务 + do_work(backend) + finally: + backend.disconnect() +``` + +#### 4. 监控连接状态 + +```python +import logging + +# 启用后端日志 +logging.getLogger('rhosocial.activerecord.backend').setLevel(logging.DEBUG) + +# 后端会自动记录连接恢复事件 +# DEBUG: Connection lost, reconnecting... +# DEBUG: Reconnected successfully +``` + +### 连接错误 SQLSTATE 代码参考 + +| SQLSTATE | 名称 | 描述 | +|----------|------|------| +| 08000 | CONNECTION_EXCEPTION | 一般连接异常 | +| 08001 | SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION | 客户端无法建立连接 | +| 08003 | CONNECTION_DOES_NOT_EXIST | 连接不存在 | +| 08004 | SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION | 服务器拒绝连接 | +| 08006 | CONNECTION_FAILURE | 连接失败 | +| 08007 | TRANSACTION_RESOLUTION_UNKNOWN | 事务解决状态未知 | +| 08P01 | PROTOCOL_VIOLATION | 协议违规 | +| 57P01 | ADMIN_SHUTDOWN | 管理员命令关闭 | +| 57P02 | CRASH_SHUTDOWN | 服务器崩溃 | +| 57P03 | CANNOT_CONNECT_NOW | 当前无法连接 | +| 57P04 | DATABASE_DROPPED | 数据库已删除 | + +💡 *AI 提示词：* "PostgreSQL 后端如何自动从连接丢失中恢复？双层保护机制是什么？" diff --git a/src/rhosocial/activerecord/backend/impl/postgres/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/__init__.py index 131e5a70..f53168c0 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/__init__.py @@ -22,6 +22,7 @@ from .dialect import PostgresDialect from .transaction import PostgresTransactionManager, AsyncPostgresTransactionManager from .types import PostgresEnumType +from .explain import PostgresExplainResult, PostgresExplainPlanLine from .statements import ( CreateEnumTypeExpression, DropEnumTypeExpression, @@ -187,6 +188,9 @@ # PostgreSQL-specific Type Helpers "PostgresEnumType", "PostgresEnumAdapter", + # PostgreSQL EXPLAIN Result Types + "PostgresExplainResult", + "PostgresExplainPlanLine", # PostgreSQL DDL Statements "CreateEnumTypeExpression", "DropEnumTypeExpression", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py index 1521d555..8aa14b70 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/async_backend.py @@ -30,6 +30,7 @@ from rhosocial.activerecord.backend.base import AsyncStorageBackend from rhosocial.activerecord.backend.introspection.backend_mixin import IntrospectorBackendMixin +from rhosocial.activerecord.backend.explain import AsyncExplainBackendMixin from rhosocial.activerecord.backend.errors import ( ConnectionError, DatabaseError, @@ -60,7 +61,7 @@ from ..transaction import AsyncPostgresTransactionManager -class AsyncPostgresBackend(IntrospectorBackendMixin, PostgresBackendMixin, AsyncStorageBackend): +class AsyncPostgresBackend(AsyncExplainBackendMixin, IntrospectorBackendMixin, PostgresBackendMixin, AsyncStorageBackend): """Asynchronous PostgreSQL-specific backend implementation.""" def __init__(self, **kwargs): @@ -344,8 +345,19 @@ async def disconnect(self): raise OperationalError(f"Error during PostgreSQL disconnection: {str(e)}") from None async def _get_cursor(self): - """Get a database cursor, ensuring connection is active.""" + """Get a database cursor, ensuring connection is active. + + This method implements automatic connection health checking (Plan A): + - Checks if connection object exists + - Checks if connection is still valid using closed/broken attributes + - Automatically reconnects if connection was lost + """ if not self._connection: + self.log(logging.DEBUG, "No connection, connecting...") + await self.connect() + elif self._connection.closed or self._connection.broken: + self.log(logging.DEBUG, "Connection lost, reconnecting...") + self._connection = None # Clear the closed/broken connection await self.connect() return self._connection.cursor() @@ -506,8 +518,173 @@ async def _detect_extensions(self) -> Dict[str, PostgresExtensionInfo]: return extensions - async def execute(self, sql: str, params: Optional[Tuple] = None, *, options=None, **kwargs) -> QueryResult: - """Execute a SQL statement with optional parameters asynchronously.""" + async def ping(self, reconnect: bool = True) -> bool: + """ + Ping the PostgreSQL server to check if the async connection is alive. + + Args: + reconnect: If True, attempt to reconnect if the connection is dead. + If False, just return the current connection status. + + Returns: + True if the connection is alive (or was successfully reconnected), + False if the connection is dead and reconnect is False or reconnection failed. + """ + try: + if not self._connection: + if reconnect: + await self.connect() + return True + else: + return False + + # Check connection status without triggering auto-reconnect + if self._connection.closed or self._connection.broken: + if reconnect: + self._connection = None + await self.connect() + return True + else: + return False + + # Execute a simple query to verify connection is actually working + cursor = await self._get_cursor() + await cursor.execute("SELECT 1") + await cursor.fetchone() + await cursor.close() + + return True + except psycopg.Error as e: + self.log(logging.WARNING, f"PostgreSQL async connection ping failed: {str(e)}") + if reconnect: + try: + self._connection = None + await self.connect() + return True + except Exception as connect_error: + self.log(logging.ERROR, f"Failed to reconnect after ping failure: {str(connect_error)}") + return False + return False + + # PostgreSQL connection error SQLSTATE codes + # Reference: https://www.postgresql.org/docs/current/errcodes-appendix.html + CONNECTION_ERROR_SQLSTATES = { + "08000", # CONNECTION_EXCEPTION + "08001", # SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION + "08003", # CONNECTION_DOES_NOT_EXIST + "08004", # SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION + "08006", # CONNECTION_FAILURE + "08007", # TRANSACTION_RESOLUTION_UNKNOWN + "08P01", # PROTOCOL_VIOLATION + "57P01", # ADMIN_SHUTDOWN + "57P02", # CRASH_SHUTDOWN + "57P03", # CANNOT_CONNECT_NOW + "57P04", # DATABASE_DROPPED + } + + def _is_connection_error(self, error: Exception) -> bool: + """ + Check if an error indicates a connection loss. + + This method identifies errors that result from lost or invalid connections, + which should trigger automatic reconnection attempts. + + Args: + error: The exception to check + + Returns: + True if the error indicates a connection problem, False otherwise + """ + if isinstance(error, PsycopgOperationalError): + # Check SQLSTATE code if available + sqlstate = getattr(error, 'sqlstate', None) + if sqlstate and sqlstate in self.CONNECTION_ERROR_SQLSTATES: + return True + + # Check for common connection error patterns in message + error_msg = str(error).lower() + connection_error_patterns = [ + 'connection', + 'connect', + 'closed', + 'terminated', + 'unexpectedly', + 'broken pipe', + 'reset by peer', + 'server closed', + 'server has gone away', + ] + for pattern in connection_error_patterns: + if pattern in error_msg: + return True + + return False + + def _is_connection_error_message(self, error_msg: str) -> bool: + """ + Check if an error message indicates a connection loss. + + This method identifies error messages that result from lost or invalid connections, + which should trigger automatic reconnection attempts. + + Args: + error_msg: The error message to check + + Returns: + True if the message indicates a connection problem, False otherwise + """ + error_msg_lower = error_msg.lower() + connection_error_patterns = [ + 'connection', + 'connect', + 'closed', + 'terminated', + 'terminating connection due to administrator command', + 'unexpectedly', + 'broken pipe', + 'reset by peer', + 'server closed', + 'server has gone away', + 'ssl connection has been closed', + 'consuming input failed', + ] + for pattern in connection_error_patterns: + if pattern in error_msg_lower: + return True + return False + + async def _reconnect(self) -> bool: + """ + Attempt to reconnect to the PostgreSQL server. + + Returns: + True if reconnection was successful, False otherwise + """ + try: + self._connection = None + await self.connect() + return True + except Exception as e: + self.log(logging.ERROR, f"Failed to reconnect: {str(e)}") + return False + + async def execute(self, sql: str, params: Optional[Tuple] = None, *, options=None, max_retries: int = 2, **kwargs) -> QueryResult: + """ + Execute a SQL statement with optional parameters asynchronously. + + This method wraps the base execute method with automatic reconnection + logic for connection errors (Plan B: Error Retry Mechanism). + + Args: + sql: SQL statement to execute + params: Optional parameters for the SQL statement + options: Execution options + max_retries: Maximum number of retry attempts for connection errors + **kwargs: Additional keyword arguments + + Returns: + QueryResult containing the query results + """ from rhosocial.activerecord.backend.options import ExecutionOptions, StatementType # If no options provided, create default options from kwargs @@ -527,7 +704,7 @@ async def execute(self, sql: str, params: Optional[Tuple] = None, *, options=Non options = ExecutionOptions( stmt_type=stmt_type, - process_result_set=None, # Let the base logic determine this based on stmt_type + process_result_set=None, column_adapters=column_adapters, column_mapping=column_mapping, ) @@ -539,36 +716,23 @@ async def execute(self, sql: str, params: Optional[Tuple] = None, *, options=Non if "column_adapters" in kwargs: options.column_adapters = kwargs["column_adapters"] - return await super().execute(sql, params, options=options) - - async def ping(self, reconnect: bool = True) -> bool: - """Ping the PostgreSQL server to check if the async connection is alive.""" - try: - if not self._connection: - if reconnect: - await self.connect() - return True - else: - return False - - # Execute a simple query to check connection - cursor = await self._get_cursor() - await cursor.execute("SELECT 1") - await cursor.fetchone() - await cursor.close() - - return True - except psycopg.Error as e: - self.log(logging.WARNING, f"PostgreSQL async connection ping failed: {str(e)}") - if reconnect: - try: - await self.disconnect() - await self.connect() - return True - except Exception as connect_error: - self.log(logging.ERROR, f"Failed to reconnect after ping failure: {str(connect_error)}") - return False - return False + # Retry loop for connection errors (Plan B) + # We catch the framework's OperationalError because the original PsycopgOperationalError + # has already been converted by _handle_error() in the base class. + for attempt in range(max_retries + 1): + try: + return await super().execute(sql, params, options=options) + except OperationalError as e: + error_msg = str(e) + if self._is_connection_error_message(error_msg) and attempt < max_retries: + self.log(logging.WARNING, + f"Connection error on attempt {attempt + 1}, retrying... Error: {error_msg}") + if await self._reconnect(): + continue + # If reconnection fails, break and raise the error + self.log(logging.ERROR, f"Failed to reconnect after connection error") + break + raise async def _handle_error(self, error: Exception) -> None: """Handle PostgreSQL-specific errors asynchronously.""" @@ -674,5 +838,11 @@ def log(self, level: int, message: str): # Fallback logging print(f"[{logging.getLevelName(level)}] {message}") + def _parse_explain_result(self, raw_rows, sql, duration): + """Return a typed :class:`PostgresExplainResult` for PostgreSQL EXPLAIN output.""" + from ..explain import PostgresExplainResult, PostgresExplainPlanLine + rows = [PostgresExplainPlanLine(line=r.get("QUERY PLAN", "")) for r in raw_rows] + return PostgresExplainResult(raw_rows=raw_rows, sql=sql, duration=duration, rows=rows) + __all__ = ["AsyncPostgresBackend"] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py index 43cb2216..9f69e8a5 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py @@ -29,6 +29,7 @@ from rhosocial.activerecord.backend.base import StorageBackend from rhosocial.activerecord.backend.introspection.backend_mixin import IntrospectorBackendMixin +from rhosocial.activerecord.backend.explain import SyncExplainBackendMixin from rhosocial.activerecord.backend.errors import ( ConnectionError, DatabaseError, @@ -60,7 +61,7 @@ from ..introspection import SyncPostgreSQLIntrospector -class PostgresBackend(IntrospectorBackendMixin, PostgresBackendMixin, StorageBackend): +class PostgresBackend(SyncExplainBackendMixin, IntrospectorBackendMixin, PostgresBackendMixin, StorageBackend): """PostgreSQL-specific backend implementation.""" def __init__(self, **kwargs): @@ -345,8 +346,19 @@ def disconnect(self): raise OperationalError(f"Error during PostgreSQL disconnection: {str(e)}") from None def _get_cursor(self): - """Get a database cursor, ensuring connection is active.""" + """Get a database cursor, ensuring connection is active. + + This method implements automatic connection health checking (Plan A): + - Checks if connection object exists + - Checks if connection is still valid using closed/broken attributes + - Automatically reconnects if connection was lost + """ if not self._connection: + self.log(logging.DEBUG, "No connection, connecting...") + self.connect() + elif self._connection.closed or self._connection.broken: + self.log(logging.DEBUG, "Connection lost, reconnecting...") + self._connection = None # Clear the closed/broken connection self.connect() return self._connection.cursor() @@ -507,8 +519,173 @@ def _detect_extensions(self) -> Dict[str, PostgresExtensionInfo]: return extensions - def execute(self, sql: str, params: Optional[Tuple] = None, *, options=None, **kwargs) -> QueryResult: - """Execute a SQL statement with optional parameters.""" + def ping(self, reconnect: bool = True) -> bool: + """ + Ping the PostgreSQL server to check if the connection is alive. + + Args: + reconnect: If True, attempt to reconnect if the connection is dead. + If False, just return the current connection status. + + Returns: + True if the connection is alive (or was successfully reconnected), + False if the connection is dead and reconnect is False or reconnection failed. + """ + try: + if not self._connection: + if reconnect: + self.connect() + return True + else: + return False + + # Check connection status without triggering auto-reconnect + if self._connection.closed or self._connection.broken: + if reconnect: + self._connection = None + self.connect() + return True + else: + return False + + # Execute a simple query to verify connection is actually working + cursor = self._get_cursor() + cursor.execute("SELECT 1") + cursor.fetchone() + cursor.close() + + return True + except psycopg.Error as e: + self.log(logging.WARNING, f"PostgreSQL connection ping failed: {str(e)}") + if reconnect: + try: + self._connection = None + self.connect() + return True + except Exception as connect_error: + self.log(logging.ERROR, f"Failed to reconnect after ping failure: {str(connect_error)}") + return False + return False + + # PostgreSQL connection error SQLSTATE codes + # Reference: https://www.postgresql.org/docs/current/errcodes-appendix.html + CONNECTION_ERROR_SQLSTATES = { + "08000", # CONNECTION_EXCEPTION + "08001", # SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION + "08003", # CONNECTION_DOES_NOT_EXIST + "08004", # SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION + "08006", # CONNECTION_FAILURE + "08007", # TRANSACTION_RESOLUTION_UNKNOWN + "08P01", # PROTOCOL_VIOLATION + "57P01", # ADMIN_SHUTDOWN + "57P02", # CRASH_SHUTDOWN + "57P03", # CANNOT_CONNECT_NOW + "57P04", # DATABASE_DROPPED + } + + def _is_connection_error(self, error: Exception) -> bool: + """ + Check if an error indicates a connection loss. + + This method identifies errors that result from lost or invalid connections, + which should trigger automatic reconnection attempts. + + Args: + error: The exception to check + + Returns: + True if the error indicates a connection problem, False otherwise + """ + if isinstance(error, PsycopgOperationalError): + # Check SQLSTATE code if available + sqlstate = getattr(error, 'sqlstate', None) + if sqlstate and sqlstate in self.CONNECTION_ERROR_SQLSTATES: + return True + + # Check for common connection error patterns in message + error_msg = str(error).lower() + connection_error_patterns = [ + 'connection', + 'connect', + 'closed', + 'terminated', + 'unexpectedly', + 'broken pipe', + 'reset by peer', + 'server closed', + 'server has gone away', + ] + for pattern in connection_error_patterns: + if pattern in error_msg: + return True + + return False + + def _reconnect(self) -> bool: + """ + Attempt to reconnect to the PostgreSQL server. + + Returns: + True if reconnection was successful, False otherwise + """ + try: + self._connection = None + self.connect() + return True + except Exception as e: + self.log(logging.ERROR, f"Failed to reconnect: {str(e)}") + return False + + def _is_connection_error_message(self, error_msg: str) -> bool: + """ + Check if an error message indicates a connection loss. + + This method identifies error messages that result from lost or invalid connections, + which should trigger automatic reconnection attempts. + + Args: + error_msg: The error message to check + + Returns: + True if the message indicates a connection problem, False otherwise + """ + error_msg_lower = error_msg.lower() + connection_error_patterns = [ + 'connection', + 'connect', + 'closed', + 'terminated', + 'terminating connection due to administrator command', + 'unexpectedly', + 'broken pipe', + 'reset by peer', + 'server closed', + 'server has gone away', + 'ssl connection has been closed', + 'consuming input failed', + ] + for pattern in connection_error_patterns: + if pattern in error_msg_lower: + return True + return False + + def execute(self, sql: str, params: Optional[Tuple] = None, *, options=None, max_retries: int = 2, **kwargs) -> QueryResult: + """ + Execute a SQL statement with optional parameters. + + This method wraps the base execute method with automatic reconnection + logic for connection errors (Plan B: Error Retry Mechanism). + + Args: + sql: SQL statement to execute + params: Optional parameters for the SQL statement + options: Execution options + max_retries: Maximum number of retry attempts for connection errors + **kwargs: Additional keyword arguments + + Returns: + QueryResult containing the query results + """ from rhosocial.activerecord.backend.options import ExecutionOptions, StatementType # If no options provided, create default options from kwargs @@ -528,7 +705,7 @@ def execute(self, sql: str, params: Optional[Tuple] = None, *, options=None, **k options = ExecutionOptions( stmt_type=stmt_type, - process_result_set=None, # Let the base logic determine this based on stmt_type + process_result_set=None, column_adapters=column_adapters, column_mapping=column_mapping, ) @@ -540,36 +717,23 @@ def execute(self, sql: str, params: Optional[Tuple] = None, *, options=None, **k if "column_adapters" in kwargs: options.column_adapters = kwargs["column_adapters"] - return super().execute(sql, params, options=options) - - def ping(self, reconnect: bool = True) -> bool: - """Ping the PostgreSQL server to check if the connection is alive.""" - try: - if not self._connection: - if reconnect: - self.connect() - return True - else: - return False - - # Execute a simple query to check connection - cursor = self._get_cursor() - cursor.execute("SELECT 1") - cursor.fetchone() - cursor.close() - - return True - except psycopg.Error as e: - self.log(logging.WARNING, f"PostgreSQL connection ping failed: {str(e)}") - if reconnect: - try: - self.disconnect() - self.connect() - return True - except Exception as connect_error: - self.log(logging.ERROR, f"Failed to reconnect after ping failure: {str(connect_error)}") - return False - return False + # Retry loop for connection errors (Plan B) + # We catch the framework's OperationalError because the original PsycopgOperationalError + # has already been converted by _handle_error() in the base class. + for attempt in range(max_retries + 1): + try: + return super().execute(sql, params, options=options) + except OperationalError as e: + error_msg = str(e) + if self._is_connection_error_message(error_msg) and attempt < max_retries: + self.log(logging.WARNING, + f"Connection error on attempt {attempt + 1}, retrying... Error: {error_msg}") + if self._reconnect(): + continue + # If reconnection fails, break and raise the error + self.log(logging.ERROR, f"Failed to reconnect after connection error") + break + raise def _handle_error(self, error: Exception) -> None: """Handle PostgreSQL-specific errors.""" @@ -675,5 +839,11 @@ def log(self, level: int, message: str): # Fallback logging print(f"[{logging.getLevelName(level)}] {message}") + def _parse_explain_result(self, raw_rows, sql, duration): + """Return a typed :class:`PostgresExplainResult` for PostgreSQL EXPLAIN output.""" + from ..explain import PostgresExplainResult, PostgresExplainPlanLine + rows = [PostgresExplainPlanLine(line=r.get("QUERY PLAN", "")) for r in raw_rows] + return PostgresExplainResult(raw_rows=raw_rows, sql=sql, duration=duration, rows=rows) + __all__ = ["PostgresBackend"] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/dialect.py b/src/rhosocial/activerecord/backend/impl/postgres/dialect.py index d7287b6d..697b3b94 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/dialect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/dialect.py @@ -147,6 +147,7 @@ CreateMaterializedViewExpression, DropMaterializedViewExpression, RefreshMaterializedViewExpression, + ExplainExpression, ) @@ -396,6 +397,39 @@ def supports_explain_format(self, format_type: str) -> bool: supported_formats = ["TEXT", "XML", "JSON", "YAML"] return format_type_upper in supported_formats + def format_explain_statement(self, explain_expr: "ExplainExpression") -> tuple: + """Build the PostgreSQL EXPLAIN SQL string and return (sql, params). + + PostgreSQL syntax: ``EXPLAIN [ ( option [, ...] ) ] statement`` + + Supported options assembled here: + - ``ANALYZE`` + - ``FORMAT { TEXT | XML | JSON | YAML }`` + - ``VERBOSE`` (passed through ``ExplainOptions.verbose`` if present) + """ + from rhosocial.activerecord.backend.expression.statements import ExplainType + + statement_sql, statement_params = explain_expr.statement.to_sql() + options = explain_expr.options + if options is None: + return f"EXPLAIN {statement_sql}", statement_params + + opts: list = [] + + if options.analyze: + opts.append("ANALYZE") + + if options.format is not None: + fmt_name = options.format.name if hasattr(options.format, "name") else str(options.format) + opts.append(f"FORMAT {fmt_name.upper()}") + elif options.type is not None and options.type == ExplainType.QUERY_PLAN: + # PostgreSQL has no QUERY PLAN keyword; plain EXPLAIN is equivalent + pass + + if opts: + return "EXPLAIN (" + ", ".join(opts) + ") " + statement_sql, statement_params + return f"EXPLAIN {statement_sql}", statement_params + def supports_graph_match(self) -> bool: """Whether graph query MATCH clause is supported.""" # PostgreSQL doesn't have native MATCH clause like some other systems diff --git a/src/rhosocial/activerecord/backend/impl/postgres/explain/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/explain/__init__.py new file mode 100644 index 00000000..15ba783c --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/explain/__init__.py @@ -0,0 +1,10 @@ +# src/rhosocial/activerecord/backend/impl/postgres/explain/__init__.py +"""PostgreSQL-specific EXPLAIN result types.""" + +from .types import IndexUsage, PostgresExplainPlanLine, PostgresExplainResult + +__all__ = [ + "IndexUsage", + "PostgresExplainPlanLine", + "PostgresExplainResult", +] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/explain/types.py b/src/rhosocial/activerecord/backend/impl/postgres/explain/types.py new file mode 100644 index 00000000..afd2edda --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/explain/types.py @@ -0,0 +1,97 @@ +# src/rhosocial/activerecord/backend/impl/postgres/explain/types.py +""" +PostgreSQL-specific EXPLAIN result types. + +PostgreSQL's EXPLAIN output (default TEXT format) produces one row per +plan line, each row having a single column named ``QUERY PLAN``. The +text in that column uses indentation and keywords like ``Seq Scan``, +``Index Scan``, and ``Index Only Scan`` to describe the query plan. +""" +from typing import List + +from pydantic import BaseModel + +from rhosocial.activerecord.backend.explain.types import BaseExplainResult + +try: + from typing import Literal +except ImportError: # Python 3.8 + from typing_extensions import Literal # type: ignore[assignment] + +# --------------------------------------------------------------------------- +# Index-usage label (mirrors SQLite / MySQL for a consistent API) +# --------------------------------------------------------------------------- +IndexUsage = Literal["full_scan", "index_with_lookup", "covering_index", "unknown"] + + +class PostgresExplainPlanLine(BaseModel): + """One line from PostgreSQL's plain-text EXPLAIN output. + + The column returned by psycopg is named ``"QUERY PLAN"`` (case-sensitive). + The field is aliased as ``line`` for convenience. + """ + + line: str + + model_config = {"populate_by_name": True} + + +class PostgresExplainResult(BaseExplainResult): + """PostgreSQL EXPLAIN result (default TEXT format). + + Contains index-usage analysis helpers that mirror the SQLite and MySQL + equivalents for a consistent cross-backend API. + + Analysis is performed via text-matching on the plan lines, looking for + PostgreSQL-specific keywords: + - ``Seq Scan`` → full table scan + - ``Index Scan`` → index lookup with table access + - ``Index Only Scan`` → covering index (no heap access) + - ``Bitmap`` patterns → treated as index_with_lookup + """ + + rows: List[PostgresExplainPlanLine] = [] + + # ------------------------------------------------------------------ + # Index-usage analysis + # ------------------------------------------------------------------ + + def analyze_index_usage(self) -> IndexUsage: + """Analyse index usage from the EXPLAIN output lines. + + Returns one of: ``"full_scan"``, ``"index_with_lookup"``, + ``"covering_index"``, or ``"unknown"``. + """ + if not self.rows: + return "unknown" + + combined = " ".join(r.line for r in self.rows).upper() + + # Covering index is the most specific — check first + if "INDEX ONLY SCAN" in combined: + return "covering_index" + + # Any index scan (with heap access) + if "INDEX SCAN" in combined or "BITMAP INDEX SCAN" in combined or "BITMAP HEAP SCAN" in combined: + return "index_with_lookup" + + # Full sequential scan + if "SEQ SCAN" in combined: + return "full_scan" + + return "unknown" + + @property + def is_full_scan(self) -> bool: + """``True`` when no index is used (sequential scan).""" + return self.analyze_index_usage() == "full_scan" + + @property + def is_index_used(self) -> bool: + """``True`` when any index is used.""" + return self.analyze_index_usage() in ("index_with_lookup", "covering_index") + + @property + def is_covering_index(self) -> bool: + """``True`` when an index-only scan eliminates heap access.""" + return self.analyze_index_usage() == "covering_index" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py index 0fde141a..ad2852bb 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py @@ -142,3 +142,79 @@ async def async_postgres_backend(request): backend.scenario_name = scenario_name # Attach for test reference yield backend await provider.async_cleanup() + + +# --- Non-parameterized Single Backend Fixtures --- + + +@pytest.fixture(scope="function") +def postgres_backend_single(): + """Non-parameterized fixture using the first available scenario.""" + scenario_names = get_scenario_names() + if not scenario_names: + pytest.skip("No PostgreSQL scenarios configured") + scenario_name = scenario_names[0] + provider = BackendFeatureProvider() + backend = provider.setup_backend(scenario_name) + backend.scenario_name = scenario_name + yield backend + provider.cleanup() + + +@pytest_asyncio.fixture(scope="function") +async def async_postgres_backend_single(): + """Non-parameterized async fixture using the first available scenario.""" + scenario_names = get_scenario_names() + if not scenario_names: + pytest.skip("No PostgreSQL scenarios configured") + scenario_name = scenario_names[0] + provider = BackendFeatureProvider() + backend = await provider.setup_async_backend(scenario_name) + backend.scenario_name = scenario_name + yield backend + await provider.async_cleanup() + + +# --- Control Backend for Session Modification Tests --- + + +@pytest.fixture(scope="function") +def postgres_control_backend(): + """ + Dedicated control backend for tests that modify PostgreSQL session settings. + + This fixture provides an independent backend instance for operations that + need to control or interfere with the main test backend, such as: + - Terminating other connections (pg_terminate_backend) + - Setting global variables + - Monitoring other connections + + The fixture is NOT parameterized to ensure consistent behavior across + all PostgreSQL scenarios. + """ + scenario_names = get_scenario_names() + if not scenario_names: + pytest.skip("No PostgreSQL scenarios configured") + scenario_name = scenario_names[0] + provider = BackendFeatureProvider() + backend = provider.setup_backend(scenario_name) + yield backend + provider.cleanup() + + +@pytest_asyncio.fixture(scope="function") +async def async_postgres_control_backend(): + """ + Dedicated async control backend for tests that modify PostgreSQL session settings. + + This fixture provides an independent async backend instance for operations that + need to control or interfere with the main test backend. + """ + scenario_names = get_scenario_names() + if not scenario_names: + pytest.skip("No PostgreSQL scenarios configured") + scenario_name = scenario_names[0] + provider = BackendFeatureProvider() + backend = await provider.setup_async_backend(scenario_name) + yield backend + await provider.async_cleanup() diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_backend_explain.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_backend_explain.py new file mode 100644 index 00000000..d35c6621 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_backend_explain.py @@ -0,0 +1,282 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_backend_explain.py +""" +Integration tests for PostgresBackend.explain() and AsyncPostgresBackend.explain(). + +These tests require a real PostgreSQL connection configured via postgres_scenarios.yaml. +The tests create temporary tables, run EXPLAIN, and verify the typed result objects. +""" +import pytest +import pytest_asyncio + +from rhosocial.activerecord.backend.explain import ( + SyncExplainBackendProtocol, + AsyncExplainBackendProtocol, +) +from rhosocial.activerecord.backend.expression import RawSQLExpression +from rhosocial.activerecord.backend.expression.statements import ExplainOptions +from rhosocial.activerecord.backend.impl.postgres import ( + PostgresExplainResult, + PostgresExplainPlanLine, +) + + +# --------------------------------------------------------------------------- +# Schema helpers (PostgreSQL syntax) +# --------------------------------------------------------------------------- + +_SETUP_STMTS = [ + "DROP TABLE IF EXISTS explain_order_items CASCADE", + "DROP TABLE IF EXISTS explain_orders CASCADE", + """CREATE TABLE explain_orders ( + id SERIAL PRIMARY KEY, + status VARCHAR(20) NOT NULL, + amount NUMERIC(10,2) + )""", + "CREATE INDEX idx_orders_status ON explain_orders(status)", + """CREATE TABLE explain_order_items ( + id SERIAL PRIMARY KEY, + order_id INTEGER NOT NULL, + sku VARCHAR(50) NOT NULL, + qty INTEGER NOT NULL DEFAULT 1 + )""", + "CREATE INDEX idx_items_order_id_sku ON explain_order_items(order_id, sku)", + """INSERT INTO explain_orders (status, amount) VALUES + ('pending', 10.00), ('pending', 20.00), + ('shipped', 30.00), ('delivered', 40.00)""", + """INSERT INTO explain_order_items (order_id, sku, qty) VALUES + (1, 'A001', 1), (1, 'A002', 2), + (2, 'B001', 1), (3, 'A001', 3)""", +] + +_CLEANUP_STMTS = [ + "DROP TABLE IF EXISTS explain_order_items CASCADE", + "DROP TABLE IF EXISTS explain_orders CASCADE", +] + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="function") +def indexed_backend(postgres_backend): + """Sync backend with test tables and indexes.""" + for stmt in _SETUP_STMTS: + postgres_backend.execute(stmt) + yield postgres_backend + try: + for stmt in _CLEANUP_STMTS: + postgres_backend.execute(stmt) + except Exception: + pass + + +@pytest_asyncio.fixture(scope="function") +async def async_indexed_backend(async_postgres_backend): + """Async backend with test tables and indexes.""" + for stmt in _SETUP_STMTS: + await async_postgres_backend.execute(stmt) + yield async_postgres_backend + try: + for stmt in _CLEANUP_STMTS: + await async_postgres_backend.execute(stmt) + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Protocol checking +# --------------------------------------------------------------------------- + +class TestExplainProtocol: + def test_sync_backend_implements_protocol(self, postgres_backend): + assert isinstance(postgres_backend, SyncExplainBackendProtocol) + + @pytest.mark.asyncio + async def test_async_backend_implements_protocol(self, async_postgres_backend): + assert isinstance(async_postgres_backend, AsyncExplainBackendProtocol) + + +# --------------------------------------------------------------------------- +# Sync explain – basic structure +# --------------------------------------------------------------------------- + +class TestSyncExplainBasic: + def test_explain_returns_postgres_explain_result(self, indexed_backend): + dialect = indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = indexed_backend.explain(expr) + assert isinstance(result, PostgresExplainResult) + + def test_result_has_rows(self, indexed_backend): + dialect = indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = indexed_backend.explain(expr) + assert len(result.rows) > 0 + + def test_result_row_type(self, indexed_backend): + dialect = indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = indexed_backend.explain(expr) + for row in result.rows: + assert isinstance(row, PostgresExplainPlanLine) + assert isinstance(row.line, str) + assert len(row.line) > 0 + + def test_result_sql_starts_with_explain(self, indexed_backend): + dialect = indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = indexed_backend.explain(expr) + assert result.sql.upper().startswith("EXPLAIN") + assert "explain_orders" in result.sql.lower() + + def test_result_has_duration(self, indexed_backend): + dialect = indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = indexed_backend.explain(expr) + assert result.duration >= 0.0 + + def test_result_has_raw_rows(self, indexed_backend): + dialect = indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = indexed_backend.explain(expr) + assert isinstance(result.raw_rows, list) + assert len(result.raw_rows) == len(result.rows) + # Each raw row must have the 'QUERY PLAN' key + for raw in result.raw_rows: + assert "QUERY PLAN" in raw + + +# --------------------------------------------------------------------------- +# Sync explain – index usage analysis +# --------------------------------------------------------------------------- + +class TestSyncExplainIndexAnalysis: + def test_full_scan_detection(self, indexed_backend): + """SELECT * FROM table without WHERE → Seq Scan → full_scan.""" + dialect = indexed_backend.dialect + # ANALYZE forces actual execution so the planner won't cheat + result = indexed_backend.explain( + RawSQLExpression(dialect, "SELECT * FROM explain_orders") + ) + assert result.analyze_index_usage() == "full_scan" + assert result.is_full_scan is True + assert result.is_index_used is False + assert result.is_covering_index is False + + def test_index_used_detection(self, indexed_backend): + """SELECT * … WHERE indexed_col = ? → Index Scan (with heap access).""" + dialect = indexed_backend.dialect + result = indexed_backend.explain( + RawSQLExpression(dialect, "SELECT * FROM explain_orders WHERE status = 'pending'") + ) + usage = result.analyze_index_usage() + # Small table may still use Seq Scan; if it uses an index, verify the flag + if usage != "full_scan": + assert result.is_index_used is True + assert result.is_full_scan is False + + def test_covering_index_detection(self, indexed_backend): + """SELECT indexed_cols WHERE indexed_col = ? → Index Only Scan.""" + dialect = indexed_backend.dialect + result = indexed_backend.explain( + RawSQLExpression( + dialect, + "SELECT order_id, sku FROM explain_order_items WHERE order_id = 1" + ) + ) + usage = result.analyze_index_usage() + # PostgreSQL may choose Index Only Scan for covered queries + assert usage in ("covering_index", "index_with_lookup", "full_scan") + if usage == "covering_index": + assert result.is_covering_index is True + + def test_plan_lines_contain_text(self, indexed_backend): + """Plan lines should contain meaningful plan text.""" + dialect = indexed_backend.dialect + result = indexed_backend.explain( + RawSQLExpression(dialect, "SELECT * FROM explain_orders") + ) + combined = " ".join(r.line for r in result.rows) + # Should mention the table name or a scan type + assert any(kw in combined.upper() for kw in ("SCAN", "EXPLAIN_ORDERS", "SEQ")) + + +# --------------------------------------------------------------------------- +# Sync explain – ANALYZE option +# --------------------------------------------------------------------------- + +class TestSyncExplainAnalyze: + def test_explain_analyze_returns_result(self, indexed_backend): + """EXPLAIN (ANALYZE) executes the query and returns runtime stats.""" + dialect = indexed_backend.dialect + if not dialect.supports_explain_analyze(): + pytest.skip("Dialect does not support EXPLAIN ANALYZE") + opts = ExplainOptions(analyze=True) + result = indexed_backend.explain( + RawSQLExpression(dialect, "SELECT * FROM explain_orders"), + opts, + ) + assert isinstance(result, PostgresExplainResult) + assert len(result.rows) > 0 + assert "ANALYZE" in result.sql.upper() + # ANALYZE output typically includes "actual time" + combined = " ".join(r.line for r in result.rows).lower() + assert "actual" in combined or "time" in combined + + def test_explain_analyze_sql_prefix(self, indexed_backend): + """Verify the generated SQL uses bracket syntax: EXPLAIN (ANALYZE).""" + dialect = indexed_backend.dialect + if not dialect.supports_explain_analyze(): + pytest.skip("Dialect does not support EXPLAIN ANALYZE") + opts = ExplainOptions(analyze=True) + result = indexed_backend.explain( + RawSQLExpression(dialect, "SELECT 1"), + opts, + ) + assert result.sql.upper().startswith("EXPLAIN (ANALYZE)") + + +# --------------------------------------------------------------------------- +# Async explain – mirror of sync tests +# --------------------------------------------------------------------------- + +class TestAsyncExplainBasic: + @pytest.mark.asyncio + async def test_explain_returns_postgres_explain_result(self, async_indexed_backend): + dialect = async_indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = await async_indexed_backend.explain(expr) + assert isinstance(result, PostgresExplainResult) + + @pytest.mark.asyncio + async def test_result_has_rows(self, async_indexed_backend): + dialect = async_indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = await async_indexed_backend.explain(expr) + assert len(result.rows) > 0 + + @pytest.mark.asyncio + async def test_result_sql_and_duration(self, async_indexed_backend): + dialect = async_indexed_backend.dialect + expr = RawSQLExpression(dialect, "SELECT * FROM explain_orders") + result = await async_indexed_backend.explain(expr) + assert result.sql.upper().startswith("EXPLAIN") + assert result.duration >= 0.0 + + @pytest.mark.asyncio + async def test_full_scan_detection(self, async_indexed_backend): + dialect = async_indexed_backend.dialect + result = await async_indexed_backend.explain( + RawSQLExpression(dialect, "SELECT * FROM explain_orders") + ) + assert result.is_full_scan is True + + @pytest.mark.asyncio + async def test_raw_rows_have_query_plan_key(self, async_indexed_backend): + dialect = async_indexed_backend.dialect + result = await async_indexed_backend.explain( + RawSQLExpression(dialect, "SELECT * FROM explain_orders") + ) + for raw in result.raw_rows: + assert "QUERY PLAN" in raw diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_connection_resilience.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_connection_resilience.py new file mode 100644 index 00000000..93be3c45 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_connection_resilience.py @@ -0,0 +1,340 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_connection_resilience.py +""" +Connection Resilience Tests + +Tests for PostgreSQL backend's ability to handle connection loss scenarios: +1. Connection terminated by pg_terminate_backend() +2. Automatic reconnection via Plan A (pre-query check) and Plan B (error retry) +3. Manual ping reconnection + +These tests verify the implementation of: +- Plan A: Pre-query connection check in _get_cursor() +- Plan B: Error retry mechanism in execute() + +Both synchronous (PostgresBackend) and asynchronous (AsyncPostgresBackend) are tested. + +Note: Unlike MySQL's is_connected() which actively checks connection state, +psycopg v3's closed/broken attributes only update after an operation fails. +Therefore, tests focus on actual query execution and reconnection behavior. +""" +import pytest +import pytest_asyncio +import asyncio +import time +import logging + +from rhosocial.activerecord.backend.impl.postgres import PostgresBackend, AsyncPostgresBackend + + +logger = logging.getLogger(__name__) + + +def print_separator(title: str): + """Print a visual separator for test sections.""" + print("\n" + "=" * 70) + print(f" {title}") + print("=" * 70) + + +# ============================================================================ +# Synchronous Backend Tests +# ============================================================================ + + +class TestConnectionStateAttributes: + """Tests for the closed/broken attributes in various states.""" + + def test_connection_not_closed_when_connected(self, postgres_backend_single: PostgresBackend): + """Verify closed and broken are False for active connection.""" + assert postgres_backend_single._connection is not None + assert postgres_backend_single._connection.closed is False + assert postgres_backend_single._connection.broken is False + + def test_connection_closed_after_disconnect(self, postgres_backend_single: PostgresBackend): + """Verify closed is True after explicit disconnect.""" + postgres_backend_single.disconnect() + # After disconnect, _connection should be None + assert postgres_backend_single._connection is None + + +class TestTerminateConnectionRecovery: + """Tests for pg_terminate_backend() recovery via Plan A + Plan B.""" + + def test_terminate_triggers_reconnection( + self, + postgres_backend_single: PostgresBackend, + postgres_control_backend: PostgresBackend + ): + """Test that pg_terminate_backend() triggers automatic reconnection.""" + print_separator("Test: pg_terminate_backend Recovery") + + # 1. Get current connection PID + result = postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + original_pid = result.data[0]['pid'] + print(f"Original connection PID: {original_pid}") + + # 2. Terminate the connection + print(f"Terminating connection {original_pid}...") + postgres_control_backend.execute(f"SELECT pg_terminate_backend({original_pid})") + time.sleep(1) + + # 3. Execute query - should trigger reconnection (Plan A or Plan B) + print("Executing query after connection terminated...") + result = postgres_backend_single.execute("SELECT 1 AS test") + print(f"Query succeeded: {result}") + assert result.data[0]['test'] == 1 + + # 4. Verify new connection PID + new_pid_result = postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + new_pid = new_pid_result.data[0]['pid'] + print(f"New connection PID: {new_pid}") + assert new_pid != original_pid + + def test_multiple_queries_after_reconnection( + self, + postgres_backend_single: PostgresBackend, + postgres_control_backend: PostgresBackend + ): + """Test that multiple queries work correctly after reconnection.""" + print_separator("Test: Multiple Queries After Reconnection") + + # Terminate connection + result = postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + original_pid = result.data[0]['pid'] + postgres_control_backend.execute(f"SELECT pg_terminate_backend({original_pid})") + time.sleep(1) + + # Execute multiple queries + for i in range(5): + result = postgres_backend_single.execute(f"SELECT {i} AS value") + assert result.data[0]['value'] == i + print(f"Query {i + 1}/5 succeeded") + + +class TestIdleTimeoutRecovery: + """Tests for idle_in_transaction_session_timeout recovery.""" + + def test_idle_timeout_triggers_reconnection( + self, + postgres_backend_single: PostgresBackend, + postgres_control_backend: PostgresBackend + ): + """ + Test that idle_in_transaction_session_timeout triggers automatic reconnection. + + Note: We test idle_in_transaction_session_timeout instead of statement_timeout + because it's more relevant to connection pooling scenarios. + """ + print_separator("Test: idle_in_transaction_session_timeout Recovery") + + # 1. Get original timeout + result = postgres_backend_single.execute( + "SHOW idle_in_transaction_session_timeout" + ) + original_timeout = result.data[0]['idle_in_transaction_session_timeout'] + print(f"Original idle_in_transaction_session_timeout: {original_timeout}") + + # 2. Set short timeout (5 seconds) in a transaction + test_timeout = "5s" + print(f"Setting idle_in_transaction_session_timeout to {test_timeout}...") + postgres_backend_single.execute( + f"SET SESSION idle_in_transaction_session_timeout = '{test_timeout}'" + ) + + # Verify setting + result = postgres_backend_single.execute( + "SHOW idle_in_transaction_session_timeout" + ) + current_timeout = result.data[0]['idle_in_transaction_session_timeout'] + print(f"Current timeout: {current_timeout}") + + # 3. Wait for timeout + print("Waiting 7 seconds for timeout...") + time.sleep(7) + + # 4. Execute query - should trigger reconnection + print("Executing query after timeout...") + try: + result = postgres_backend_single.execute("SELECT 1 AS test") + print(f"Query succeeded: {result}") + assert result.data[0]['test'] == 1 + finally: + # Restore original timeout via control backend + try: + postgres_control_backend.execute( + f"SET GLOBAL idle_in_transaction_session_timeout = '{original_timeout}'" + ) + except Exception: + pass # Ignore cleanup errors + + +class TestPingReconnect: + """Tests for manual ping reconnection.""" + + def test_ping_reconnect_after_terminate( + self, + postgres_backend_single: PostgresBackend, + postgres_control_backend: PostgresBackend + ): + """Test that ping(reconnect=True) can restore a terminated connection.""" + print_separator("Test: Ping Reconnect") + + # 1. Get current connection PID + result = postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + original_pid = result.data[0]['pid'] + print(f"Original connection PID: {original_pid}") + + # 2. Terminate the connection + print(f"Terminating connection {original_pid}...") + postgres_control_backend.execute(f"SELECT pg_terminate_backend({original_pid})") + time.sleep(1) + + # 3. Ping with reconnect - should succeed + print("Calling ping(reconnect=True)...") + ping_result = postgres_backend_single.ping(reconnect=True) + print(f"Ping result: {ping_result}") + assert ping_result is True, "Ping should succeed with reconnect=True" + + # 4. Verify connection is restored + assert postgres_backend_single._connection is not None + assert postgres_backend_single._connection.closed is False + + # 5. Verify new connection PID + new_pid_result = postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + new_pid = new_pid_result.data[0]['pid'] + print(f"New connection PID: {new_pid}") + assert new_pid != original_pid + + def test_ping_alive_connection_returns_true(self, postgres_backend_single: PostgresBackend): + """Test that ping returns True for alive connections.""" + print_separator("Test: Ping Alive Connection") + + ping_result = postgres_backend_single.ping(reconnect=False) + print(f"Ping result for alive connection: {ping_result}") + assert ping_result is True + + +# ============================================================================ +# Asynchronous Backend Tests +# ============================================================================ + + +class TestAsyncConnectionStateAttributes: + """Async tests for the closed/broken attributes in various states.""" + + @pytest.mark.asyncio + async def test_connection_not_closed_when_connected(self, async_postgres_backend_single: AsyncPostgresBackend): + """Verify closed and broken are False for active connection.""" + assert async_postgres_backend_single._connection is not None + assert async_postgres_backend_single._connection.closed is False + assert async_postgres_backend_single._connection.broken is False + + @pytest.mark.asyncio + async def test_connection_closed_after_disconnect(self, async_postgres_backend_single: AsyncPostgresBackend): + """Verify closed is True after explicit disconnect.""" + await async_postgres_backend_single.disconnect() + # After disconnect, _connection should be None + assert async_postgres_backend_single._connection is None + + +class TestAsyncTerminateConnectionRecovery: + """Async tests for pg_terminate_backend() recovery.""" + + @pytest.mark.asyncio + async def test_terminate_triggers_reconnection( + self, + async_postgres_backend_single: AsyncPostgresBackend, + async_postgres_control_backend: AsyncPostgresBackend + ): + """Test that pg_terminate_backend() triggers automatic reconnection.""" + print_separator("Test: Async pg_terminate_backend Recovery") + + # 1. Get current connection PID + result = await async_postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + original_pid = result.data[0]['pid'] + print(f"Original connection PID: {original_pid}") + + # 2. Terminate the connection + print(f"Terminating connection {original_pid}...") + await async_postgres_control_backend.execute(f"SELECT pg_terminate_backend({original_pid})") + await asyncio.sleep(1) + + # 3. Execute query - should trigger reconnection + print("Executing query after connection terminated...") + result = await async_postgres_backend_single.execute("SELECT 1 AS test") + print(f"Query succeeded: {result}") + assert result.data[0]['test'] == 1 + + # 4. Verify new connection PID + new_pid_result = await async_postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + new_pid = new_pid_result.data[0]['pid'] + print(f"New connection PID: {new_pid}") + assert new_pid != original_pid + + @pytest.mark.asyncio + async def test_multiple_queries_after_reconnection( + self, + async_postgres_backend_single: AsyncPostgresBackend, + async_postgres_control_backend: AsyncPostgresBackend + ): + """Test that multiple queries work correctly after reconnection.""" + print_separator("Test: Async Multiple Queries After Reconnection") + + # Terminate connection + result = await async_postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + original_pid = result.data[0]['pid'] + await async_postgres_control_backend.execute(f"SELECT pg_terminate_backend({original_pid})") + await asyncio.sleep(1) + + # Execute multiple queries + for i in range(5): + result = await async_postgres_backend_single.execute(f"SELECT {i} AS value") + assert result.data[0]['value'] == i + print(f"Query {i + 1}/5 succeeded") + + +class TestAsyncPingReconnect: + """Async tests for manual ping reconnection.""" + + @pytest.mark.asyncio + async def test_ping_reconnect_after_terminate( + self, + async_postgres_backend_single: AsyncPostgresBackend, + async_postgres_control_backend: AsyncPostgresBackend + ): + """Test that ping(reconnect=True) can restore a terminated connection.""" + print_separator("Test: Async Ping Reconnect") + + # 1. Get current connection PID + result = await async_postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + original_pid = result.data[0]['pid'] + print(f"Original connection PID: {original_pid}") + + # 2. Terminate the connection + print(f"Terminating connection {original_pid}...") + await async_postgres_control_backend.execute(f"SELECT pg_terminate_backend({original_pid})") + await asyncio.sleep(1) + + # 3. Ping with reconnect - should succeed + print("Calling ping(reconnect=True)...") + ping_result = await async_postgres_backend_single.ping(reconnect=True) + print(f"Ping result: {ping_result}") + assert ping_result is True, "Ping should succeed with reconnect=True" + + # 4. Verify connection is restored + assert async_postgres_backend_single._connection.closed is False + + # 5. Verify new connection PID + new_pid_result = await async_postgres_backend_single.execute("SELECT pg_backend_pid() AS pid") + new_pid = new_pid_result.data[0]['pid'] + print(f"New connection PID: {new_pid}") + assert new_pid != original_pid + + @pytest.mark.asyncio + async def test_ping_alive_connection_returns_true(self, async_postgres_backend_single: AsyncPostgresBackend): + """Test that ping returns True for alive connections.""" + print_separator("Test: Async Ping Alive Connection") + + ping_result = await async_postgres_backend_single.ping(reconnect=False) + print(f"Ping result for alive connection: {ping_result}") + assert ping_result is True From b318ac7a74b5f1979eb92a446cd581ce37773c19 Mon Sep 17 00:00:00 2001 From: zhao vistart <864657+vistart@users.noreply.github.com> Date: Thu, 2 Apr 2026 23:32:39 +0800 Subject: [PATCH 04/11] feat(cli): improve info command with connection status and add introspect usage examples (#17) * feat(cli): improve info command with connection status and add introspect usage examples - Display connection status in info command output - Show "(from actual connection)" when connected to database - Show "(default value - no database connection)" when not connected - Use actual dialect from backend instead of creating new instance - Display installed extensions when connected - Add detailed usage examples to introspect subcommand help - Examples for all introspection types (tables, views, columns, etc.) - JSON output example - Schema parameter example (PostgreSQL specific) - Environment variables connection example Co-Authored-By: Claude Opus 4.6 * chore: add changelog fragment for PR #17 --------- Co-authored-by: Claude Opus 4.6 --- changelog.d/17.added.md | 1 + .../backend/impl/postgres/__main__.py | 323 ++++++++++++------ 2 files changed, 217 insertions(+), 107 deletions(-) create mode 100644 changelog.d/17.added.md diff --git a/changelog.d/17.added.md b/changelog.d/17.added.md new file mode 100644 index 00000000..20b0db81 --- /dev/null +++ b/changelog.d/17.added.md @@ -0,0 +1 @@ +Improved PostgreSQL backend CLI info command with connection status display and added comprehensive introspect usage examples. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/__main__.py b/src/rhosocial/activerecord/backend/impl/postgres/__main__.py index 387ba4d2..a0843e25 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/__main__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/__main__.py @@ -188,90 +188,173 @@ def parse_args(): # ========================================================================= # Design Notes: # ========================================================================= - # Uses explicit subcommand mode: query and introspect are mutually exclusive. + # Uses explicit subcommand mode: info, query and introspect are peers. # This avoids argparse misidentifying SQL queries as subcommand names. # # Connection parameters are shared between subcommands and placed in parent parser. - # The main parser also inherits from parent, so global options like --info - # can access these parameters. + # Only subcommand parsers inherit from parent, main parser does not. # ========================================================================= # Parent parser: shared connection parameters parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument("--host", default=os.getenv("POSTGRES_HOST", "localhost"), help="Database host") - parent_parser.add_argument("--port", type=int, default=int(os.getenv("POSTGRES_PORT", "5432")), help="Database port") - parent_parser.add_argument("--database", default=os.getenv("POSTGRES_DATABASE"), help="Database name (optional for some operations)") - parent_parser.add_argument("--user", default=os.getenv("POSTGRES_USER", "postgres"), help="Database user") - parent_parser.add_argument("--password", default=os.getenv("POSTGRES_PASSWORD", ""), help="Database password") parent_parser.add_argument( - "--output", - choices=["table", "json", "csv", "tsv"], - default="table", - help='Output format. Defaults to "table" if rich is installed.', + "--host", + default=os.getenv("POSTGRES_HOST", "localhost"), + help="Database host (env: POSTGRES_HOST, default: localhost)", + ) + parent_parser.add_argument( + "--port", + type=int, + default=int(os.getenv("POSTGRES_PORT", "5432")), + help="Database port (env: POSTGRES_PORT, default: 5432)", + ) + parent_parser.add_argument( + "--database", + default=os.getenv("POSTGRES_DATABASE"), + help="Database name (env: POSTGRES_DATABASE, optional for some operations)", + ) + parent_parser.add_argument( + "--user", + default=os.getenv("POSTGRES_USER", "postgres"), + help="Database user (env: POSTGRES_USER, default: postgres)", + ) + parent_parser.add_argument( + "--password", + default=os.getenv("POSTGRES_PASSWORD", ""), + help="Database password (env: POSTGRES_PASSWORD)", + ) + parent_parser.add_argument( + "--use-async", + action="store_true", + help="Use asynchronous backend", ) - parent_parser.add_argument("--log-level", default="INFO", help="Set logging level (e.g., DEBUG, INFO)") - parent_parser.add_argument("--rich-ascii", action="store_true", help="Use ASCII characters for rich table borders.") - parent_parser.add_argument("--use-async", action="store_true", help="Use asynchronous backend") parent_parser.add_argument( "--version", type=str, default=None, help='PostgreSQL version to simulate (e.g., "15.0.0"). Default: auto-detect.', ) + parent_parser.add_argument( + "-o", "--output", + choices=["table", "json", "csv", "tsv"], + default="table", + help='Output format. Defaults to "table" if rich is installed.', + ) + parent_parser.add_argument( + "--log-level", + default="INFO", + help="Set logging level (e.g., DEBUG, INFO)", + ) + parent_parser.add_argument( + "--rich-ascii", + action="store_true", + help="Use ASCII characters for rich table borders.", + ) - # Main parser inherits from parent, so --info can access shared parameters + # Main parser does NOT inherit from parent parser = argparse.ArgumentParser( description="Execute SQL queries against a PostgreSQL backend.", formatter_class=argparse.RawTextHelpFormatter, - parents=[parent_parser] ) - # Global options (no subcommand required) - parser.add_argument("--info", action="store_true", help="Display PostgreSQL environment information.") + # Global options parser.add_argument( - "-v", "--verbose", action="count", default=0, help="Increase verbosity. -v for families, -vv for details." + "-v", "--verbose", + action="count", + default=0, + help="Increase verbosity. -v for families, -vv for details.", ) - # Subcommands: query and introspect - subparsers = parser.add_subparsers(dest='command', help='Available commands') + # Subcommands: info, query and introspect + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # info subcommand + info_parser = subparsers.add_parser( + "info", + help="Display PostgreSQL environment information", + parents=[parent_parser], + ) # query subcommand query_parser = subparsers.add_parser( - 'query', - help='Execute SQL query', - parents=[parent_parser] + "query", + help="Execute SQL query", + parents=[parent_parser], + ) + query_parser.add_argument( + "sql", + nargs="?", + default=None, + help="SQL query to execute. If not provided, reads from --file or stdin.", ) query_parser.add_argument( - "sql", nargs="?", default=None, help="SQL query to execute. If not provided, reads from --file or stdin." + "-f", "--file", + default=None, + help="Path to a file containing SQL to execute.", ) - query_parser.add_argument("-f", "--file", default=None, help="Path to a file containing SQL to execute.") # introspect subcommand introspect_parser = subparsers.add_parser( - 'introspect', - help='Database introspection commands', - parents=[parent_parser] + "introspect", + help="Database introspection commands", + parents=[parent_parser], + epilog="""Examples: + # List all tables in database + %(prog)s tables --database mydb + + # List all views + %(prog)s views --database mydb + + # Get detailed table info (columns, indexes, foreign keys) + %(prog)s table users --database mydb + + # Get column details for a table + %(prog)s columns users --database mydb + + # Get index information + %(prog)s indexes users --database mydb + + # Get foreign key relationships + %(prog)s foreign-keys users --database mydb + + # List triggers + %(prog)s triggers --database mydb + + # Get database information + %(prog)s database --database mydb + + # Output as JSON + %(prog)s tables --database mydb -o json + + # Specify schema (PostgreSQL specific) + %(prog)s tables --database mydb --schema public + + # Using environment variables for connection + export POSTGRES_HOST=localhost POSTGRES_DATABASE=mydb POSTGRES_USER=postgres POSTGRES_PASSWORD=secret + %(prog)s tables +""", + formatter_class=argparse.RawDescriptionHelpFormatter, ) introspect_parser.add_argument( - 'type', + "type", choices=INTROSPECT_TYPES, - help='Introspection type: tables, views, table, columns, indexes, foreign-keys, triggers, database' + help="Introspection type: tables, views, table, columns, indexes, foreign-keys, triggers, database", ) introspect_parser.add_argument( - 'name', - nargs='?', + "name", + nargs="?", default=None, - help='Table/view name (required for table, columns, indexes, foreign-keys types)' + help="Table/view name (required for table, columns, indexes, foreign-keys types)", ) introspect_parser.add_argument( - '--schema', + "--schema", default=None, - help='Schema name (defaults to "public" for PostgreSQL)' + help='Schema name (defaults to "public" for PostgreSQL)', ) introspect_parser.add_argument( - '--include-system', - action='store_true', - help='Include system tables in output' + "--include-system", + action="store_true", + help="Include system tables in output", ) return parser.parse_args() @@ -373,33 +456,68 @@ def parse_version(version_str: str) -> Tuple[int, int, int]: return (major, minor, patch) -def display_info( - verbose: int = 0, - output_format: str = "table", - version_str: Optional[str] = None, - extensions: Optional[Dict[str, PostgresExtensionInfo]] = None, -): - """Display PostgreSQL environment information.""" - # Parse version - if version_str: - version = parse_version(version_str) - else: - version = (13, 0, 0) # Default version +def handle_info(args, provider: Any): + """Handle info subcommand. - dialect = PostgresDialect(version=version) + Display PostgreSQL environment information based on actual database connection + or default values if no database is provided. + """ + output_format = args.output if args.output != "table" or RICH_AVAILABLE else "json" - # Set extensions if provided - if extensions: - dialect._extensions = extensions # type: ignore + # Track whether we're using actual database or defaults + is_connected = False + dialect = None + extensions = None + version_display = None - version_display = f"{version[0]}.{version[1]}.{version[2]}" + if args.database: + try: + config = PostgresConnectionConfig( + host=args.host, + port=args.port, + database=args.database, + username=args.user, + password=args.password, + ) + backend = PostgresBackend(connection_config=config) + backend.connect() + backend.introspect_and_adapt() + + # Use the adapted dialect from backend + dialect = backend.dialect + version_tuple = backend.get_server_version() + if version_tuple: + version_display = f"{version_tuple[0]}.{version_tuple[1]}.{version_tuple[2]}" + + # Get extensions from dialect + if hasattr(backend.dialect, "_extensions"): + extensions = backend.dialect._extensions + is_connected = True + + backend.disconnect() + except Exception as e: + logger.warning("Could not connect to database for introspection: %s", e) + logger.warning("Using default values for dialect information.") + # Fall back to command-line version or default + + # Create default dialect if not connected + if dialect is None: + # Parse version from command line or use default + actual_version = args.version + if actual_version: + version = parse_version(actual_version) + else: + version = (13, 0, 0) # Default version + dialect = PostgresDialect(version=version) + version_display = f"{version[0]}.{version[1]}.{version[2]}" # Unified structure for JSON output info = { "database": { "type": "postgresql", "version": version_display, - "version_tuple": list(version), + "version_tuple": list(dialect.version), + "connected": is_connected, }, "features": { "extensions": {}, @@ -439,7 +557,7 @@ def display_info( if value: supported_count += 1 - if verbose >= 2: + if args.verbose >= 2: info["protocols"][group_name][protocol_name] = { "supported": supported_count, "total": total_count, @@ -462,15 +580,27 @@ def display_info( "extensions": info["features"]["extensions"], "protocols": info["protocols"], } - _display_info_rich(info_legacy, verbose, version_display, extensions) + _display_info_rich(info_legacy, args.verbose, version_display, extensions, is_connected) return info def _display_info_rich( - info: Dict, verbose: int, version_display: str, extensions: Optional[Dict[str, PostgresExtensionInfo]] + info: Dict, + verbose: int, + version_display: str, + extensions: Optional[Dict[str, PostgresExtensionInfo]], + is_connected: bool = True, ): - """Display info using rich console.""" + """Display info using rich console. + + Args: + info: Information dictionary containing database and protocol info + verbose: Verbosity level for output detail + version_display: Version string for display + extensions: Optional dictionary of extension information + is_connected: Whether the info is from actual database connection + """ from rich.console import Console console = Console(force_terminal=True) @@ -481,7 +611,11 @@ def _display_info_rich( console.print("\n[bold cyan]PostgreSQL Environment Information[/bold cyan]\n") - console.print(f"[bold]PostgreSQL Version:[/bold] {version_display}\n") + # Show connection status + if is_connected: + console.print(f"[bold]PostgreSQL Version:[/bold] {version_display} [dim](from actual connection)[/dim]\n") + else: + console.print(f"[bold]PostgreSQL Version:[/bold] {version_display} [yellow](default value - no database connection)[/yellow]\n") # Display extensions if available if extensions: @@ -794,58 +928,31 @@ async def handle_introspect_async(args, backend: AsyncPostgresBackend, provider: def main(): args = parse_args() - # Handle --info flag (global option, no subcommand needed) - if args.info: - output_format = args.output if args.output != "table" or RICH_AVAILABLE else "json" - - # Try to connect and get real version/extensions if database is provided - extensions = None - actual_version = args.version - - if args.database: - try: - config = PostgresConnectionConfig( - host=args.host, port=args.port, database=args.database, username=args.user, password=args.password - ) - backend = PostgresBackend(connection_config=config) - backend.connect() - backend.introspect_and_adapt() - - # Get actual version - version_tuple = backend.get_server_version() - if version_tuple: - actual_version = f"{version_tuple[0]}.{version_tuple[1]}.{version_tuple[2]}" - - # Get extensions from dialect - if hasattr(backend.dialect, "_extensions"): - extensions = backend.dialect._extensions - - backend.disconnect() - except Exception as e: - logger.warning("Could not connect to database for introspection: %s", e) - # Fall back to command-line version or default - - display_info( - verbose=args.verbose, output_format=output_format, version_str=actual_version, extensions=extensions - ) - return - - # Require a subcommand if --info is not specified + # Must specify a subcommand if args.command is None: - print("Error: Please specify a command: 'query' or 'introspect'", file=sys.stderr) + print("Error: Please specify a command: 'info', 'query', or 'introspect'", file=sys.stderr) print("Use --help for more information.", file=sys.stderr) sys.exit(1) + provider = get_provider(args) + + # Handle info subcommand + if args.command == "info": + handle_info(args, provider) + return + # Handle introspect subcommand if args.command == "introspect": if not args.database: print("Error: --database is required for introspection", file=sys.stderr) sys.exit(1) - provider = get_provider(args) config = PostgresConnectionConfig( - host=args.host, port=args.port, database=args.database, - username=args.user, password=args.password + host=args.host, + port=args.port, + database=args.database, + username=args.user, + password=args.password, ) if args.use_async: @@ -861,8 +968,6 @@ def main(): if not isinstance(numeric_level, int): raise ValueError(f"Invalid log level: {args.log_level}") - provider = get_provider(args) - if RICH_AVAILABLE and isinstance(provider, RichOutputProvider): from rich.console import Console @@ -897,7 +1002,11 @@ def main(): sys.exit(1) config = PostgresConnectionConfig( - host=args.host, port=args.port, database=args.database, username=args.user, password=args.password + host=args.host, + port=args.port, + database=args.database, + username=args.user, + password=args.password, ) kwargs = {"use_ascii": args.rich_ascii} From 8a7f11c0ca95eb1a6854ef40147289eb0d413edd Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 5 Apr 2026 15:01:31 +0800 Subject: [PATCH 05/11] feat(test,docs): add parallel worker processing support - Add WorkerTestProtocol implementation to BasicProvider and QueryProvider - Add supports_for_update() method to PostgresDialect - Add worker pool tests for basic and query features: - Worker pool lifecycle management - Parallel CRUD operations - Connection management - Parallel query execution - Transaction isolation - Add parallel workers scenario documentation (EN/CN) - Add parallel workers example code - Update CI to use git branch for testsuite installation --- .github/workflows/test.yml | 2 +- docs/en_US/README.md | 3 + docs/en_US/scenarios/README.md | 27 + docs/en_US/scenarios/parallel_workers.md | 636 +++++++++++++++++ .../parallel_workers/README.md | 222 ++++++ .../parallel_workers/config_loader.py | 114 +++ .../exp1_basic_multiprocess.py | 291 ++++++++ .../exp2_postgres_async_advantage.py | 256 +++++++ .../parallel_workers/exp3_deadlock_wrong.py | 173 +++++ .../exp4_partition_correct.py | 412 +++++++++++ .../exp5_multithread_warning.py | 189 +++++ .../parallel_workers/models.py | 192 +++++ .../parallel_workers/setup_db.py | 202 ++++++ docs/zh_CN/README.md | 3 + docs/zh_CN/scenarios/README.md | 27 + docs/zh_CN/scenarios/parallel_workers.md | 653 ++++++++++++++++++ .../backend/impl/postgres/dialect.py | 10 + tests/providers/basic.py | 72 +- tests/providers/query.py | 91 ++- .../feature/basic/worker/__init__.py | 2 + .../feature/basic/worker/conftest.py | 9 + .../worker/test_connection_management.py | 10 + .../basic/worker/test_parallel_crud.py | 11 + .../worker/test_worker_pool_lifecycle.py | 10 + .../feature/query/worker/__init__.py | 2 + .../feature/query/worker/conftest.py | 9 + .../query/worker/test_parallel_queries.py | 11 + .../worker/test_transaction_isolation.py | 11 + 28 files changed, 3647 insertions(+), 3 deletions(-) create mode 100644 docs/en_US/scenarios/README.md create mode 100644 docs/en_US/scenarios/parallel_workers.md create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/README.md create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/config_loader.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/models.py create mode 100644 docs/examples/chapter_08_scenarios/parallel_workers/setup_db.py create mode 100644 docs/zh_CN/scenarios/README.md create mode 100644 docs/zh_CN/scenarios/parallel_workers.md create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_parallel_crud.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_parallel_queries.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a06c5b22..bfa249ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: - name: Install activerecord and testsuite from PyPI run: | pip install git+https://github.com/rhosocial/python-activerecord@release/v1.0.0.dev21 - pip install rhosocial-activerecord-testsuite==1.0.0.dev7 + pip install git+https://github.com/rhosocial/python-activerecord-testsuite@release/v1.0.0.dev8 - name: Create postgres scenario config run: | diff --git a/docs/en_US/README.md b/docs/en_US/README.md index 4df51d4e..10326350 100644 --- a/docs/en_US/README.md +++ b/docs/en_US/README.md @@ -48,4 +48,7 @@ * **[Performance Issues](troubleshooting/performance.md)**: performance bottleneck analysis * **[SQL Standard Compliance](troubleshooting/sql_standard.md)**: strict SQL compliance handling +8. **[Scenarios](scenarios/README.md)** + * **[Parallel Worker Processing](scenarios/parallel_workers.md)**: correct usage in multi-process/async concurrent scenarios + > 📖 **Core Library Documentation**: To learn about the complete functionality of the ActiveRecord framework, please refer to [rhosocial-activerecord documentation](https://github.com/rhosocial/python-activerecord/tree/main/docs/en_US). diff --git a/docs/en_US/scenarios/README.md b/docs/en_US/scenarios/README.md new file mode 100644 index 00000000..27e73262 --- /dev/null +++ b/docs/en_US/scenarios/README.md @@ -0,0 +1,27 @@ +# Scenarios + +This chapter provides complete application examples of the PostgreSQL backend in real-world business scenarios, helping developers understand how to correctly use rhosocial-activerecord in actual projects. + +## Scenario List + +### [Parallel Worker Processing](parallel_workers.md) + +Demonstrates correct usage of PostgreSQL backend in multi-process/async concurrent scenarios: + +- **Multi-process Correct Usage**: `configure()` must be called within child processes +- **PostgreSQL Async Advantage**: psycopg native async support, true network I/O concurrency +- **Deadlock Handling**: PostgreSQL automatically detects deadlocks, recommended retry mechanism for production +- **Multi-threading Pitfalls**: psycopg connections also do not support multi-threaded concurrency + +> 📖 **Companion Code**: Complete runnable experiment code is located in the `docs/examples/chapter_08_scenarios/parallel_workers/` directory. + +## Relationship with Core Library Scenarios + +This chapter is a PostgreSQL-specific supplement to the [Core Library Scenarios Documentation](https://github.com/Rhosocial/python-activerecord/tree/main/docs/en_US/scenarios), focusing on: + +- PostgreSQL-specific concurrent behavior (MVCC, row-level locking, deadlock detection) +- psycopg async driver characteristics (single-connection model, true async I/O) +- RETURNING clause advantages (INSERT/UPDATE/DELETE returning data) +- Key differences compared to MySQL and SQLite + +The general ActiveRecord usage patterns introduced in the core library scenarios (such as relationships, query building) also apply to the PostgreSQL backend. diff --git a/docs/en_US/scenarios/parallel_workers.md b/docs/en_US/scenarios/parallel_workers.md new file mode 100644 index 00000000..abbbbc51 --- /dev/null +++ b/docs/en_US/scenarios/parallel_workers.md @@ -0,0 +1,636 @@ +# Correct Usage in Parallel Worker Scenarios (PostgreSQL) + +In scenarios such as data processing, task queues, and batch imports, developers often want to use multiple workers to process tasks in parallel to improve throughput. This chapter focuses on parallel worker usage patterns for PostgreSQL databases, explains the fundamental differences between PostgreSQL, MySQL, and SQLite in concurrent processing, and provides verified safe solutions. + +> **Design Principle Throughout This Chapter**: The synchronous class `BaseActiveRecord` and asynchronous class `AsyncBaseActiveRecord` in `rhosocial-activerecord` have **identical method names** - `configure()` / `backend()` / `transaction()` / `save()` etc. are all the same, the async version just needs `await` or `async with`. All examples in this chapter provide both versions. + +## Table of Contents + +1. [PostgreSQL Concurrency Overview](#1-postgresql-concurrency-overview) +2. [Multi-process: Recommended Approach](#2-multi-process-recommended-approach) +3. [PostgreSQL Async Backend Characteristics](#3-postgresql-async-backend-characteristics) +4. [Deadlock: PostgreSQL Automatic Detection and Prevention](#4-deadlock-postgresql-automatic-detection-and-prevention) +5. [Application Separation Principle](#5-application-separation-principle) +6. [asyncio Concurrency Behavior on PostgreSQL](#6-asyncio-concurrency-behavior-on-postgresql) +7. [Example Code](#7-example-code) + +--- + +## 1. PostgreSQL Concurrency Overview + +### 1.1 Differences Compared to MySQL and SQLite + +`rhosocial-activerecord` follows the core design principle of **one ActiveRecord class bound to one connection**: + +- **Sync**: `Post.configure(config, PostgreSQLBackend)` → writes to `Post.__backend__` +- **Async**: `await Post.configure(config, AsyncPostgreSQLBackend)` → writes to `Post.__backend__` + +The configuration writes to a **class-level attribute**, which is still unsafe for multi-threading. PostgreSQL differs from other databases in the following ways: + +| Feature | SQLite | MySQL | PostgreSQL | +| --- | --- | --- | --- | +| Lock Granularity | File-level lock | Row-level lock (InnoDB) | Row-level lock (MVCC) | +| Concurrent Writes | Requires WAL mode | Supported by default | Supported by default (MVCC) | +| Async Driver | `aiosqlite`: thread pool simulation | `mysql-connector-python` native async | `psycopg` native async | +| Deadlock Handling | Timeout wait | Auto detection, errno 1213 | Auto detection, SQLSTATE 40P01 | +| Connection Type | File path (local) | TCP network connection | TCP network connection | +| RETURNING | ❌ Not supported | ❌ Not supported | ✅ Natively supported | +| Free-threaded | ✅ Supported | ✅ Supported | ❌ Not supported (psycopg limitation) | + +### 1.2 Immutability of Single Connection Model + +Regardless of PostgreSQL's concurrency advantages, **the `__backend__` bound to a single `ActiveRecord` class is still one connection**. In multi-threaded environments, concurrent operations on the same `__backend__` will cause cursor state corruption. The `psycopg` documentation explicitly states that connections do not support multi-threaded concurrent use. + +> **Do not share ActiveRecord configuration across multiple threads.** Multi-processing is the correct choice for parallel worker scenarios. + +### 1.3 psycopg Free-threaded Limitation + +`psycopg` version 3 does not support free-threaded builds of Python 3.13t/3.14t. If your project requires free-threaded Python, please consider using the MySQL or SQLite backend. + +--- + +## 2. Multi-process: Recommended Approach + +Multi-processing is the recommended approach for parallel worker scenarios. Each process has its own memory space, `configure()` executes independently within the process, establishing independent TCP connections. + +### 2.1 Correct Lifecycle + +**Sync (multiprocessing)**: + +```python +import multiprocessing +from rhosocial.activerecord.backend.impl.postgres import PostgreSQLBackend, PostgreSQLConnectionConfig +from models import Comment, Post, User + +def worker(post_ids: list[int]): + # 1. After process starts, configure connection within process - each process establishes independent TCP connection + config = PostgreSQLConnectionConfig( + host="localhost", + port=5432, + database="mydb", + username="app", + password="secret", + ) + User.configure(config, PostgreSQLBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + try: + for post_id in post_ids: + post = Post.find_one(post_id) + if post is None: + continue + author = post.author() # BelongsTo association + approved = len([c for c in post.comments() if c.is_approved]) + post.view_count = 1 + approved + post.save() + finally: + # 2. Disconnect before process exits + User.backend().disconnect() + + +if __name__ == "__main__": + post_ids = list(range(1, 101)) + chunk_size = 25 + + with multiprocessing.Pool(processes=4) as pool: + chunks = [post_ids[i:i+chunk_size] for i in range(0, len(post_ids), chunk_size)] + pool.map(worker, chunks) +``` + +**Async (asyncio + multi-process)**: + +```python +import asyncio +import multiprocessing +from rhosocial.activerecord.backend.impl.postgres import AsyncPostgreSQLBackend, PostgreSQLConnectionConfig +from models import AsyncComment, AsyncPost, AsyncUser + +async def async_worker_main(post_ids: list[int]): + # 1. Configure async connection within process (psycopg native async, TCP network I/O) + config = PostgreSQLConnectionConfig( + host="localhost", port=5432, + database="mydb", username="app", password="secret", + ) + await AsyncUser.configure(config, AsyncPostgreSQLBackend) + AsyncPost.__backend__ = AsyncUser.backend() + AsyncComment.__backend__ = AsyncUser.backend() + + try: + async def process_post(post_id: int): + post = await AsyncPost.find_one(post_id) + if post is None: + return + author = await post.author() # Add await and parentheses + approved = len([c for c in await post.comments() if c.is_approved]) + post.view_count = 1 + approved + await post.save() + + # Single connection sequential execution (concurrency is between processes) + for pid in post_ids: + await process_post(pid) + finally: + await AsyncUser.backend().disconnect() + + +def run_async_worker(post_ids: list[int]): + # Each process has its own event loop and independent PostgreSQL connection + asyncio.run(async_worker_main(post_ids)) + + +if __name__ == "__main__": + post_ids = list(range(1, 101)) + chunk_size = 25 + + with multiprocessing.Pool(processes=4) as pool: + chunks = [post_ids[i:i+chunk_size] for i in range(0, len(post_ids), chunk_size)] + pool.map(run_async_worker, chunks) +``` + +**Key Rules**: + +- `configure()` must be called within child processes, not configured in parent process before `fork` +- PostgreSQL connections are TCP connections, inheriting file descriptors after `fork` is more dangerous than SQLite +- In async scenarios, coroutines within the same process naturally access the database sequentially (event loop single-threaded scheduling) + +Runnable timing comparison demo at [exp1_basic_multiprocess.py](../../examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py). + +--- + +## 3. PostgreSQL Async Backend Characteristics + +### 3.1 Single Connection Model Constraints + +`rhosocial-activerecord`'s async PostgreSQL backend (`AsyncPostgreSQLBackend`) is based on `psycopg` version 3's async interface, **each ActiveRecord class binds to one connection**. This differs from connection pool solutions: + +| Feature | Single Connection ORM (This Project) | Connection Pool Solution | +| --- | --- | --- | +| In-process asyncio.gather | ❌ Not supported, raises runtime error | ✅ Supported, each coroutine uses different connection | +| Configuration Complexity | Low, one line `configure()` | High, manual connection pool management | +| Multi-process Concurrency | ✅ Each process has independent connection, true concurrency | ✅ Also supported | +| Applicable Scenarios | Batch processing, task queues, data pipelines | High-concurrency web services | + +### 3.2 psycopg Async Advantages + +`psycopg`'s async implementation is based on Python's native `asyncio`, truly releasing control while waiting for database responses: + +```text +Single connection async execution path: + Coroutine A → Send SQL → await → PostgreSQL response → Coroutine A resumes + Coroutine B → Must wait for A to complete before sending (sequential execution) + +Multi-process concurrent execution path: + Process A → Independent connection → Send SQL → await (concurrent) + Process B → Independent connection → Send SQL → await (concurrent) + (Both truly concurrent at transport layer) +``` + +### 3.3 Multi-process is the Correct Approach for Concurrency + +Although coroutines within a single process must execute sequentially, **between multiple processes** each process independently holds a connection, achieving true concurrency. + +Runnable multi-process async vs sync performance comparison at [exp2_postgres_async_advantage.py](../../examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py). + +--- + +## 4. Deadlock: PostgreSQL Automatic Detection and Prevention + +PostgreSQL has a built-in deadlock detection algorithm. When deadlock occurs, it automatically rolls back one of the transactions, allowing the other to continue. The rolled-back party receives an `OperationalError` (SQLSTATE 40P01). + +### 4.1 Root Cause of Deadlock: Inconsistent Row Lock Order + +```python +# ❌ Wrong: Different workers lock rows in opposite order +def worker_a(): + with Post.transaction(): + post1 = Post.find_one(1) # Worker A locks id=1 first + time.sleep(0.01) + post2 = Post.find_one(2) # Worker A requests id=2 (held by B) + # → PostgreSQL detects deadlock, chooses to rollback A or B + +def worker_b(): + with Post.transaction(): + post2 = Post.find_one(2) # Worker B locks id=2 first + time.sleep(0.01) + post1 = Post.find_one(1) # Worker B requests id=1 (held by A) +``` + +Anti-pattern example at [exp3_deadlock_wrong.py](../../examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py). + +### 4.2 Prevention Method 1: Fixed Lock Order + +```python +# ✅ Correct: Always lock resources in primary key ascending order +def transfer_safe(from_id: int, to_id: int, amount: float): + first_id, second_id = min(from_id, to_id), max(from_id, to_id) + with Account.transaction(): + first = Account.find_one(first_id) + second = Account.find_one(second_id) + debit, credit = (first, second) if from_id < to_id else (second, first) + debit.balance -= amount + credit.balance += amount + debit.save() + credit.save() + +# Async version (same method names, add await) +async def transfer_safe_async(from_id: int, to_id: int, amount: float): + first_id, second_id = min(from_id, to_id), max(from_id, to_id) + async with Account.transaction(): + first = await Account.find_one(first_id) + second = await Account.find_one(second_id) + debit, credit = (first, second) if from_id < to_id else (second, first) + debit.balance -= amount + credit.balance += amount + await debit.save() + await credit.save() +``` + +### 4.3 Prevention Method 2: Atomic Claim (Query + Update in Transaction) + +```python +# ✅ Correct: Atomic claim within transaction, PostgreSQL row-level lock guarantees no duplicates +def claim_posts(batch_size: int = 5) -> list: + with Post.transaction(): + pending = ( + Post.query() + .where(Post.c.status == "draft") + .order_by(Post.c.id) + .limit(batch_size) + .for_update() # PostgreSQL supports FOR UPDATE + .all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + post.save() + return pending + +# Async version (same method names, add await) +async def claim_posts_async(batch_size: int = 5) -> list: + async with AsyncPost.transaction(): + pending = await ( + AsyncPost.query() + .where(AsyncPost.c.status == "draft") + .order_by(AsyncPost.c.id) + .limit(batch_size) + .for_update() + .all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + await post.save() + return pending +``` + +### 4.3.1 Cross-Database Compatible Capability Detection Pattern + +When your code needs to be reused across multiple database backends, you should check backend capabilities before using FOR UPDATE: + +```python +def claim_posts_portable(batch_size: int = 5) -> list: + """Cross-database compatible atomic claim implementation""" + # Get backend dialect and check capability + dialect = Post.backend().dialect + supports_for_update = dialect.supports_for_update() + + with Post.transaction(): + query = ( + Post.query() + .where(Post.c.status == "draft") + .order_by(Post.c.id) + .limit(batch_size) + ) + + # Only use FOR UPDATE on supported backends + if supports_for_update: + query = query.for_update() + + pending = query.all() + + if not pending: + return [] + for post in pending: + post.status = "processing" + post.save() + return pending +``` + +**Capability Detection Design Principles**: + +| Principle | Description | +| --- | --- | +| **Don't make choices for users** | Dialect returns `False` by default; only backends that explicitly support it return `True` | +| **Dual-layer defense** | ActiveQuery layer detects when `for_update()` is called; Dialect layer checks again as safety net | +| **Explicit over implicit** | Raises `UnsupportedFeatureError` when unsupported, rather than silently ignoring | +| **User adapts** | Users choose alternative approaches after checking `supports_for_update()` | + +> **PostgreSQL Note**: PostgreSQL has supported FOR UPDATE since early versions, so `PostgreSQLDialect.supports_for_update()` always returns `True`. SQLite does not support FOR UPDATE (uses file-level locks) and returns `False`. + +### 4.4 PostgreSQL-specific Method: Catch Deadlock and Retry (Production Recommended) + +PostgreSQL has automatic deadlock detection, so you can **not prevent deadlock, but catch and retry**. This is the recommended pattern for PostgreSQL production environments: + +```python +import time + +def _is_deadlock(exc: Exception) -> bool: + """Check if exception is PostgreSQL deadlock (SQLSTATE 40P01)""" + # psycopg 3 error objects have sqlstate attribute + if hasattr(exc, 'sqlstate'): + return exc.sqlstate == '40P01' + msg = str(exc) + return '40P01' in msg or 'deadlock' in msg.lower() + + +def claim_posts_with_retry(batch_size: int = 5, max_retry: int = 3) -> list: + """Atomic claim + automatic deadlock retry""" + for attempt in range(max_retry): + try: + with Post.transaction(): + pending = ( + Post.query() + .where(Post.c.status == "draft") + .order_by(Post.c.id) + .limit(batch_size) + .for_update() + .all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + post.save() + return pending + except Exception as e: + if _is_deadlock(e) and attempt < max_retry - 1: + time.sleep(0.05 * (attempt + 1)) # Exponential backoff + continue + raise + return [] +``` + +### 4.5 Five Prevention Principles (PostgreSQL-specific Notes) + +| Principle | Description | +| --- | --- | +| **Data Partitioning** | Distribute data set by ID range or hash to workers, avoid multiple processes operating on same row | +| **Consistent Lock Order** | When involving multiple resources, request locks in fixed order (e.g., primary key ascending) | +| **Short Transactions** | Only do necessary operations in transaction, no cross I/O waits or time-consuming calculations | +| **Atomic Claim** | Query and update task status in transaction, use `FOR UPDATE` to lock | +| **Deadlock Retry** (PostgreSQL-specific) | Catch `OperationalError` (SQLSTATE 40P01) and retry | + +> **PostgreSQL vs MySQL Difference**: MySQL deadlock error code is errno 1213, PostgreSQL is SQLSTATE 40P01. Both have automatic deadlock detection. + +Runnable correct solution demo at [exp4_partition_correct.py](../../examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py). + +--- + +## 5. Application Separation Principle + +When a system contains two distinctly different workloads, it is recommended to deploy them as separate applications. + +### 5.1 Scenario Examples + +| Workload Type | Characteristics | Suitable Deployment | +| --- | --- | --- | +| Web API Service | Short requests, high concurrency, response time sensitive | FastAPI / Django + asyncio + connection pool | +| Data Analysis Batch Processing | Long running, large data volume, CPU intensive | Standalone script + multiprocessing + PostgreSQLBackend | +| Task Queue Consumer | Regular polling, independent tasks, easy horizontal scaling | Standalone worker process pool | + +### 5.2 Recommended Architecture + +```text +User requests ──→ Web application (asyncio + connection pool) + │ + └──→ Task queue (PostgreSQL database table / Redis) + │ + └──→ Background worker process pool + (Each process independent PostgreSQLBackend sync connection) +``` + +Web application receives requests and writes to task queue; worker process pool executes time-consuming tasks. + +--- + +## 6. asyncio Concurrency Behavior on PostgreSQL + +### 6.1 Single Connection Model Limitations + +`rhosocial-activerecord` follows the **one ActiveRecord class bound to one connection** design principle. This means coroutines within the same process **cannot** concurrently access the database through `asyncio.gather` - a single connection can only handle one query at a time: + +```python +# ❌ Wrong: Single connection does not support concurrent access +async def batch_update_wrong(post_ids: list[int]): + await asyncio.gather(*[ + update_one(pid) for pid in post_ids # Will raise runtime error + ]) + +# ✅ Correct: Sequential execution within single connection +async def batch_update_correct(post_ids: list[int]): + for pid in post_ids: + await update_one(pid) +``` + +### 6.2 Multi-process is the Concurrency Carrier + +The advantage of async lies **between multiple processes**: each process holds an independent PostgreSQL TCP connection, achieving true concurrency between processes. + +### 6.3 PostgreSQL Async vs MySQL Async Comparison + +| Scenario | PostgreSQL (psycopg 3) | MySQL (mysql-connector-python) | +| --- | --- | --- | +| In-process asyncio.gather | ❌ Single connection not supported | ❌ Single connection not supported | +| Multi-process concurrent writes | ✅ MVCC, no configuration needed | ✅ Row-level lock, no configuration needed | +| Deadlock handling | ✅ Auto detection, SQLSTATE 40P01 | ✅ Auto detection, errno 1213 | +| Free-threaded support | ❌ Not supported | ✅ Supported | + +--- + +## 7. Example Code + +Complete runnable examples for this chapter are located at [`docs/examples/chapter_08_scenarios/parallel_workers/`](../../examples/chapter_08_scenarios/parallel_workers/), containing the following experiments: + +| File | Content | Corresponding Section | +| --- | --- | --- | +| [`config_loader.py`](../../examples/chapter_08_scenarios/parallel_workers/config_loader.py) | Connection configuration loading (YAML / environment variables / defaults) | — | +| [`models.py`](../../examples/chapter_08_scenarios/parallel_workers/models.py) | Shared model definitions (`User`, `Post`, `Comment` sync + async versions) | — | +| [`setup_db.py`](../../examples/chapter_08_scenarios/parallel_workers/setup_db.py) | Database initialization script (sync/async modes) | — | +| [`exp1_basic_multiprocess.py`](../../examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py) | Correct multi-process usage (serial/sync multi-process/async multi-process timing comparison) | §2.1 | +| [`exp2_postgres_async_advantage.py`](../../examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py) | PostgreSQL async characteristics: single connection limitation + multi-process concurrent write comparison | §3 | +| [`exp3_deadlock_wrong.py`](../../examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py) | Row lock order conflict causing PostgreSQL deadlock (anti-pattern) | §4.1 | +| [`exp4_partition_correct.py`](../../examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py) | Data partitioning + atomic claim + deadlock retry (sync/async solutions) | §4.2–4.4 | +| [`exp5_multithread_warning.py`](../../examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py) | Multi-threaded shared connection issues (anti-pattern) | §1.2 | + +> **Note**: All example files use `rhosocial-activerecord` ORM directly, with model hierarchy `User → Post → Comment`, demonstrating sync and async equivalent usage of `HasMany` / `BelongsTo` associations. + +Before running, execute the initialization script: + +```bash +cd docs/examples/chapter_08_scenarios/parallel_workers +python setup_db.py +python exp1_basic_multiprocess.py # Run any experiment +``` + +See `README.md` in that directory for complete descriptions and expected outputs of each experiment. + +--- + +## 8. WorkerPool Testing Experience Summary + +### 8.1 Known Limitations of Async Worker Testing + +When running async tests in a `WorkerPool` multi-process environment, there are known limitations: + +#### Event Loop Cross-Process Issue + +When `WorkerPool` executes async tasks in child processes, each child process creates an independent event loop via `asyncio.run()`. However, the event loop created by the test framework (pytest-asyncio) in the main process is isolated from the child process event loops: + +```text +Main Process (pytest): + └── Event Loop A (created by pytest-asyncio) + └── Fixture: async_user_class_for_worker + └── Async backend instance bound to Loop A + +Child Process (Worker): + └── Event Loop B (created by asyncio.run()) + └── Task tries to use async backend bound to Loop A + └── Error: Task got Future attached to a different loop +``` + +**Wrong Example**: + +```python +# ❌ Wrong: Async backend created in main process fixture cannot be used in child process +async def async_worker_task(user_id, conn_params): + # conn_params contains async backend instance bound to main process event loop + # Child process trying to use it will fail + backend = conn_params['backend'] # Bound to wrong loop + user = await backend.find_one(user_id) # RuntimeError! +``` + +**Correct Approach**: + +```python +# ✅ Correct: Create new async backend instance inside child process +async def async_worker_task(user_id, conn_params): + # Only pass connection parameters (serializable), create new instance in child process + config = conn_params['config_kwargs'] + await Model.configure(config, AsyncPostgreSQLBackend) + user = await Model.find_one(user_id) + await Model.backend().disconnect() +``` + +#### Affected Test Scenarios + +| Test Type | Sync Version | Async Version | Reason | +|----------|--------------|---------------|--------| +| Parallel reads | ✅ Pass | ❌ Fail | Async backend bound to main process loop | +| Parallel updates | ✅ Pass | ❌ Fail | Same as above | +| Parallel deletes | ✅ Pass | ❌ Fail | Same as above | +| Parallel queries | ✅ Pass | ❌ Fail | Same as above | +| Transaction isolation | ✅ Pass | ❌ Fail | Same as above + transaction state cross-process issue | +| Worker lifecycle | ✅ Pass | ✅ Pass | No cross-process async operations | +| Connection management | ✅ Pass | ✅ Pass | No cross-process async operations | + +### 8.2 Test Coverage Notes + +Worker test coverage contribution to PostgreSQL backend: + +| Module | Coverage | Description | +|----------|----------|-------------| +| `backend.py` | ~32% | Sync backend core functionality | +| `async_backend.py` | ~35% | Async backend core functionality | +| `mixins/` | ~18% | DML operations | +| `dialect.py` | ~22% | SQL dialect handling | + +Sync test coverage is higher; async test coverage has uncovered code paths due to event loop issues. + +### 8.3 Production Recommendations + +1. **Sync Workers Preferred**: In multi-process Worker scenarios, sync backend (`PostgreSQLBackend`) is the more stable choice +2. **Async for Single Process**: Async backend (`AsyncPostgreSQLBackend`) works well in single-process sequential execution scenarios +3. **Avoid Cross-Process Async Instance Passing**: Only pass serializable connection parameters, create new async backend instances in child processes + +### 8.4 FOR UPDATE Capability Detection Experience + +#### Problem Background + +In WorkerPool multi-process testing, the `test_transaction_isolation.py` transfer test used the `FOR UPDATE` clause to lock rows. However, SQLite backend does not support `FOR UPDATE` (uses file-level locks), causing test failures. + +#### Solution: Capability Detection Pattern + +Following the "don't make choices for users" design principle, we implemented two-layer capability detection: + +**1. Dialect-Level Capability Declaration**: + +```python +# SQLDialectBase (default not supported) +def supports_for_update(self) -> bool: + """Default returns False; only backends that support it override this method""" + return False + +# PostgreSQLDialect (explicitly supported) +def supports_for_update(self) -> bool: + return True +``` + +**2. ActiveQuery-Level Early Detection**: + +```python +# ActiveQuery.for_update() method +def for_update(self, ...): + if not dialect.supports_for_update(): + raise UnsupportedFeatureError( + dialect.name, + "FOR UPDATE clause", + "This backend does not support row-level locking with FOR UPDATE. " + "Use dialect.supports_for_update() to check support before calling this method." + ) +``` + +**3. Dialect-Level Safety Net**: + +```python +# SQLDialectBase.format_query_statement() +if expr.for_update: + if not self.supports_for_update(): + raise UnsupportedFeatureError(...) + # Generate FOR UPDATE SQL... +``` + +#### Test Code Adaptation + +Use capability detection in test task functions to adapt to different backends: + +```python +def transfer_task(from_id: int, to_id: int, amount: float, conn_params: dict): + # Check if backend supports FOR UPDATE + supports_for_update = backend.dialect.supports_for_update() + + with Model.transaction(): + if supports_for_update: + # PostgreSQL/MySQL: Use FOR UPDATE to lock + first = Model.query().where(Model.c.id == first_id).for_update().one() + else: + # SQLite: Use regular query (relies on file locks) + first = Model.find_one({'id': first_id}) + # ... business logic +``` + +#### Design Experience Summary + +| Experience | Description | +|------------|-------------| +| **Default deny principle** | Base class returns `False`, backends must explicitly declare support | +| **Don't make choices for users** | Raises error when unsupported, rather than silently ignoring (Django/Rails approach) | +| **Dual-layer defense** | ActiveQuery provides early failure, Dialect acts as safety net | +| **User adapts** | Users choose alternative approaches after checking `supports_for_update()` | + +### 8.5 Related Files + +- Test bridge files: `tests/rhosocial/activerecord_postgres_test/feature/basic/worker/` +- Provider implementations: `tests/providers/basic.py`, `tests/providers/query.py` +- WorkerPool implementation: `rhosocial.activerecord.worker.pool` diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/README.md b/docs/examples/chapter_08_scenarios/parallel_workers/README.md new file mode 100644 index 00000000..f1083949 --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/README.md @@ -0,0 +1,222 @@ +# 并行 Worker 场景实验（PostgreSQL 版） + +本目录包含《场景实战 — 并行 Worker》章节的完整可运行实验代码，所有示例均使用真实的 `rhosocial-activerecord` ORM 操作 PostgreSQL 数据库，模型体系为： + +```text +User --has_many--> Post --has_many--> Comment +User <--belongs_to-- Post +Post <--belongs_to-- Comment +User <--belongs_to-- Comment （评论作者） +``` + +每个模型提供**同步版**（继承 `ActiveRecord`）和**异步版**（继承 `AsyncActiveRecord`），方法名完全相同，异步版仅需加 `await`。 + +## 实验列表 + +| 文件 | 主题 | 类型 | +| ---- | ---- | ---- | +| `config_loader.py` | 从 YAML / 环境变量 / 默认值加载 PostgreSQL 连接配置 | 工具 | +| `setup_db.py` | 初始化数据库和测试数据 | 工具 | +| `models.py` | 共享 ActiveRecord 模型定义（同步 + 异步） | 公共 | +| `exp1_basic_multiprocess.py` | 多进程正确用法 + 串行/并行耗时对比 | ✅ 正确示范 | +| `exp2_postgres_async_advantage.py` | PostgreSQL 异步真正优势 + 并发写入能力 | ✅ 正确示范 | +| `exp3_deadlock_wrong.py` | 行锁顺序冲突导致 PostgreSQL 死锁 | ❌ 反面教材 | +| `exp4_partition_correct.py` | 数据分区 + 原子领取 + 死锁重试 | ✅ 正确示范 | +| `exp5_multithread_warning.py` | 多线程共享 PostgreSQL 连接的危险 | ❌ 反面教材 | + +## PostgreSQL vs MySQL vs SQLite 关键差异 + +| 特性 | SQLite | MySQL | PostgreSQL | +| ---- | ------ | ----- | ---------- | +| 异步后端 | `aiosqlite`（线程池模拟，有额外开销） | `mysql-connector-python` 原生 async | `psycopg` 原生 async | +| 并发写入 | 需要 WAL 模式才能并发 | 默认行级锁，开箱即用并发 | MVCC，开箱即用并发 | +| 死锁检测 | 无自动检测，超时返回 locked | InnoDB 自动检测，回滚其中一方 | 自动检测，回滚其中一方 | +| 死锁错误码 | — | errno 1213 | SQLSTATE 40P01 | +| 线程安全 | `check_same_thread=True` 默认保护 | 无内置保护，官方明确说明不支持 | 无内置保护，官方明确说明不支持 | +| RETURNING | ❌ 不支持 | ❌ 不支持 | ✅ 原生支持 | +| Free-threaded | ✅ 支持 | ✅ 支持 | ❌ 不支持（psycopg 限制） | + +## 依赖说明 + +本实验依赖 `psycopg` 版本 3： + +```bash +pip install psycopg[binary] +``` + +> **注意**：`psycopg` 不支持 Python 3.13t/3.14t 的 free-threaded 构建。如果需要 free-threaded Python，请使用 MySQL 或 SQLite 后端。 + +## 快速开始 + +### 1. 激活虚拟环境 + +```bash +# 在 python-activerecord-postgres 项目根目录 +source .venv3.8-examples/bin/activate +# 或 +source .venv3.14-examples/bin/activate +``` + +### 2. 进入实验目录 + +```bash +cd docs/examples/chapter_08_scenarios/parallel_workers +``` + +### 3. 检查连接配置 + +实验脚本自动读取 `tests/config/postgres_scenarios.yaml`（如存在），优先使用 `postgres_16` 场景。 +也可通过环境变量覆盖： + +```bash +export PGHOST=localhost +export PGPORT=5432 +export PGDATABASE=test_db +export PGUSER=postgres +export PGPASSWORD=your_password +``` + +或使用 `--scenario` 指定 YAML 场景名： + +```bash +python setup_db.py --scenario postgres_15 +``` + +### 4. 初始化数据库 + +**每次运行实验前必须先执行此步骤**（实验脚本会修改数据库状态）： + +```bash +python setup_db.py # 同步初始化（默认） +python setup_db.py --async # 异步初始化（效果相同） +``` + +预期输出： + +```text +=== 同步初始化模式 === +连接信息： + 主机：db-dev-1-n.rho.im:15432 + 数据库：test_db + 用户：postgres + +删除旧表并重新建表… +建表完成，开始插入测试数据… + User #1: user01 + ... + 共插入 20 篇文章 + 共插入 60 条评论 + +✓ 同步初始化完成 + 用户 5 / 文章 20 / 评论 60 +``` + +### 5. 运行实验 + +各实验相互独立，每次运行前先执行 `setup_db.py` 重置数据库： + +```bash +python setup_db.py && python exp1_basic_multiprocess.py +python setup_db.py && python exp2_postgres_async_advantage.py +python setup_db.py && python exp3_deadlock_wrong.py +python setup_db.py && python exp4_partition_correct.py +python setup_db.py && python exp5_multithread_warning.py +``` + +## 实验说明 + +### exp1：基础多进程用法 + +演示最核心的原则：**`configure()` 必须在子进程内调用**。 + +对比三种处理方式的耗时： + +- **实验 A**：串行（单进程，同步 ActiveRecord） +- **实验 B**：4 进程并行（同步 ActiveRecord） +- **实验 C**：4 进程并行（异步 ActiveRecord，每进程内 asyncio 驱动） + +同时演示 `BelongsTo` 关联遍历（`post.author()`）的正确用法。 + +PostgreSQL 特别说明：父进程 `configure()` 建立 TCP 连接后 `fork`，子进程继承文件描述符，多进程共享同一 TCP socket，比 SQLite 共享文件句柄更危险。 + +### exp2：PostgreSQL 异步特点与多进程并发写入 + +说明 `psycopg` 异步后端（单连接模型）的特点： + +- **Part 1**：同进程内，同步串行 vs 异步顺序，说明单连接不能 `asyncio.gather` 并发 +- **Part 2**：多进程并发写入，PostgreSQL MVCC 无需额外配置即支持并发 + +> **注意**：`psycopg` 异步后端是单连接模型，同进程内协程不能并发。多进程间每进程独立连接，才是并发的体现。 + +### exp3：反面教材 — PostgreSQL 死锁 + +故意展示不一致的行锁顺序导致 PostgreSQL 死锁（SQLSTATE 40P01）的场景。 + +PostgreSQL 会**自动检测死锁**并回滚代价较小的事务，另一方继续执行。 +被回滚的 Worker 收到 `OperationalError`，若不处理则工作丢失。 + +预期输出（参考）： + +```text +=== 错误模式：不一致的锁顺序（PostgreSQL 死锁演示）=== + Worker 0 (✅ 成功)：... + Worker 1 (💀 死锁回滚)：... + → 死锁（PostgreSQL SQLSTATE 40P01）：... +``` + +### exp4：正确的并行方案（含 PostgreSQL 特有方案 C） + +提供五种生产可用的并行策略： + +- **方案 A 同步/异步**：按 `user_id` 分区，遍历 `HasMany` 关联统计评论数 +- **方案 B 同步/异步**：事务内原子领取（`with/async with Post.transaction()`） +- **方案 C 同步**：原子领取 + 死锁自动重试（**PostgreSQL 生产环境推荐**） + +### exp5：反面教材 — 多线程问题 + +展示 PostgreSQL 多线程陷阱： + +- **场景 1**：共享 `__backend__` — 游标状态被并发线程覆盖 +- **场景 2**：每线程 `configure()` — 类属性互相覆盖，实际仍共享同一实例 + +PostgreSQL 无 `check_same_thread` 保护，但官方同样明确不支持多线程并发共享连接。 + +## 关联关系访问说明 + +同步关联返回可调用函数，需要加 `()` 才能触发查询： + +```python +# 同步 BelongsTo +author = post.author() # ✅ 正确，返回 User 实例或 None +author = post.author # ❌ 错误，返回函数对象 + +# 同步 HasMany +comments = post.comments() # ✅ 正确，返回 list[Comment] +comments = post.comments # ❌ 错误，返回函数对象 + +# 异步 BelongsTo（加 await + 括号） +author = await post.author() # ✅ 正确 +author = await post.author # ❌ 错误 + +# 异步 HasMany +comments = await post.comments() # ✅ 正确 +comments = await post.comments # ❌ 错误 +``` + +## 常见问题 + +**Q: 运行实验时报连接拒绝（Connection refused），怎么处理？** + +A: 检查 `tests/config/postgres_scenarios.yaml` 中的连接参数，或通过环境变量 `PGHOST` / `PGPORT` / `PGUSER` / `PGPASSWORD` 配置本地 PostgreSQL 服务器。 + +**Q: 报 `pyyaml` 未安装，怎么处理？** + +A: 运行 `pip install pyyaml`，或通过环境变量直接配置连接参数（无需 YAML 文件）。 + +**Q: exp3 每次运行都触发了死锁吗？** + +A: 不一定。死锁依赖时序，`sleep(0.01)` 已增大窗口，多运行几次可复现。exp4 方案 C 的重试机制可以自动处理此情况。 + +**Q: 为什么 psycopg 不支持 free-threaded Python？** + +A: `psycopg` 版本 3 的 C 扩展模块尚未适配 Python 3.13t/3.14t 的 free-threaded 构建。如需 free-threaded 支持，请使用 MySQL 或 SQLite 后端。 diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/config_loader.py b/docs/examples/chapter_08_scenarios/parallel_workers/config_loader.py new file mode 100644 index 00000000..5748eeee --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/config_loader.py @@ -0,0 +1,114 @@ +# config_loader.py - PostgreSQL Connection Configuration Loader +# docs/examples/chapter_08_scenarios/parallel_workers/config_loader.py +""" +PostgreSQL connection configuration loading (three-level priority): + +1. tests/config/postgres_scenarios.yaml (if exists, use postgres_16 or first scenario) +2. Environment variables PGHOST / PGPORT / PGDATABASE / PGUSER / PGPASSWORD +3. Hardcoded defaults (localhost:5432/test_db/postgres/) + +Public interface: + load_config(scenario=None) → PostgreSQLConnectionConfig + list_scenarios() → list[str] +""" + +from __future__ import annotations + +import os +import sys +from typing import List, Optional + +# ─── sys.path patch (enables standalone execution) ───────────────────────────────── + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import PostgresConnectionConfig # noqa: E402 + +# ─── Default preferred scenarios (by priority) ───────────────────────────────────── + +_PREFERRED_SCENARIOS = ["postgres_17", "postgres_16", "postgres_15", "postgres_14", "postgres_13"] + + +def _find_scenarios_yaml() -> Optional[str]: + """Find postgres_scenarios.yaml in standard locations.""" + # Relative to this file: ../../../../tests/config/postgres_scenarios.yaml + here = os.path.dirname(os.path.abspath(__file__)) + candidates = [ + os.path.abspath(os.path.join(here, "..", "..", "..", "..", "tests", "config", "postgres_scenarios.yaml")), + os.environ.get("PG_ACTIVERECORD_CONFIG_PATH", ""), + ] + for path in candidates: + if path and os.path.isfile(path): + return path + return None + + +def load_config(scenario: Optional[str] = None) -> PostgresConnectionConfig: + """Load PostgreSQL connection configuration. + + Args: + scenario: Scenario name in YAML (e.g., "postgres_16"). If None, auto-select by priority. + + Returns: + PostgreSQLConnectionConfig instance. + """ + yaml_path = _find_scenarios_yaml() + if yaml_path: + try: + import yaml # type: ignore[import] + + with open(yaml_path, encoding="utf-8") as f: + data = yaml.safe_load(f) + scenarios: dict = data.get("scenarios", {}) + if scenarios: + if scenario and scenario in scenarios: + params = scenarios[scenario] + else: + # Select scenario by priority + params = None + for preferred in _PREFERRED_SCENARIOS: + if preferred in scenarios: + params = scenarios[preferred] + break + if params is None: + params = next(iter(scenarios.values())) + # Filter out None values to avoid dataclass field type conflicts + return PostgresConnectionConfig(**{k: v for k, v in params.items() if v is not None}) + except ImportError: + print("Warning: pyyaml not installed, skipping YAML config loading. Run pip install pyyaml", file=sys.stderr) + except Exception as e: + print(f"Warning: Cannot read {yaml_path}: {e}, using environment variables/defaults", file=sys.stderr) + + # Build configuration from environment variables or hardcoded defaults + # PostgreSQL environment variable naming follows libpq convention + return PostgresConnectionConfig( + host=os.environ.get("PGHOST", "localhost"), + port=int(os.environ.get("PGPORT", "5432")), + database=os.environ.get("PGDATABASE", "test_db"), + username=os.environ.get("PGUSER", "postgres"), + password=os.environ.get("PGPASSWORD", ""), + ) + + +def list_scenarios() -> List[str]: + """Return all available scenario names in YAML, or empty list if config file not found.""" + yaml_path = _find_scenarios_yaml() + if yaml_path: + try: + import yaml # type: ignore[import] + + with open(yaml_path, encoding="utf-8") as f: + data = yaml.safe_load(f) + return list(data.get("scenarios", {}).keys()) + except Exception: + pass + return [] + + +def show_active_config(config: PostgresConnectionConfig) -> None: + """Print current connection parameters (password hidden).""" + print(f" Host: {config.host}:{config.port}") + print(f" Database: {config.database}") + print(f" User: {config.username}") diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py b/docs/examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py new file mode 100644 index 00000000..f4eb3425 --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py @@ -0,0 +1,291 @@ +# exp1_basic_multiprocess.py - Correct Multiprocess Usage (with Timing Comparison) +# docs/examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py +""" +Experiment objective: Demonstrate correct approach to multiprocess parallel processing +using real PostgreSQL ActiveRecord ORM. + +Key points: + - configure() must be called within child processes, not sharing connections in parent + - Each process independently owns its own database connection + - Demonstrates both synchronous (BaseActiveRecord) and asynchronous (AsyncBaseActiveRecord) usage + - Demonstrates relationships: User --(has_many)--> Post, Post --(has_many)--> Comment + +PostgreSQL-specific notes: + - PostgreSQL connections are network connections, each configure() establishes a new TCP connection + - If parent process calls configure() then forks, child processes inherit connection file + descriptors, causing write corruption + - Correct approach: fork first (Pool creation), then configure() within child processes + +Comparison experiments: + A. Serial: Single process adds a comment to each post sequentially + B. Multiprocess (sync): 4 processes process in parallel + C. Multiprocess (async): 4 processes, each using asyncio coroutines + +How to run: + python setup_db.py # Initialize database first + python exp1_basic_multiprocess.py +""" + +from __future__ import annotations + +import asyncio +import multiprocessing +import os +import sys +import time + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import ( # noqa: E402 + AsyncPostgresBackend, + PostgresBackend, +) + +from config_loader import load_config # noqa: E402 +from models import AsyncComment, AsyncPost, AsyncUser, Comment, Post, User # noqa: E402 + +NUM_WORKERS = 4 + + +# ───────────────────────────────────────────────────────────────────────────── +# Helper: Get list of published post IDs +# ───────────────────────────────────────────────────────────────────────────── + + +def fetch_published_post_ids() -> list: + config = load_config() + Post.configure(config, PostgresBackend) + posts = Post.query().where(Post.c.status == "published").all() + Post.backend().disconnect() + return [p.id for p in posts if p.id is not None] + + +def split_chunks(items: list, n: int) -> list: + size = max(1, (len(items) + n - 1) // n) + return [items[i : i + size] for i in range(0, len(items), size)] + + +def reset_exp_comments() -> None: + """Delete comments inserted by this experiment (body starts with '[exp1]')""" + config = load_config() + Comment.configure(config, PostgresBackend) + to_delete = Comment.query().where(Comment.c.body.like("[exp1]%")).all() + for c in to_delete: + c.delete() + Comment.backend().disconnect() + + +# ───────────────────────────────────────────────────────────────────────────── +# Experiment A: Serial (Single Process) +# ───────────────────────────────────────────────────────────────────────────── + + +def run_serial(post_ids: list) -> int: + """Single process serial: Add a system comment to each post""" + config = load_config() + User.configure(config, PostgresBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + bot = User.query().order_by(User.c.id).one() + if bot is None: + return 0 + + count = 0 + for post_id in post_ids: + post = Post.find_one(post_id) + if post is None: + continue + c = Comment( + post_id=post.id, + user_id=bot.id, + body=f"[exp1] Serial comment post#{post.id} by pid={os.getpid()}", + is_approved=True, + ) + c.save() + count += 1 + + User.backend().disconnect() + return count + + +# ───────────────────────────────────────────────────────────────────────────── +# Experiment B: Multiprocess (Synchronous ActiveRecord) +# ───────────────────────────────────────────────────────────────────────────── + + +def worker_sync(post_ids: list) -> int: + """ + Child process Worker (synchronous version): + 1. Call configure() within the process, establishing independent PostgreSQL network connection + 2. Use relationships to query author and add comments + """ + # ✅ configure() called in child process, each process establishes independent TCP connection + config = load_config() + User.configure(config, PostgresBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + bot = User.query().order_by(User.c.id).one() + if bot is None: + return 0 + + count = 0 + for post_id in post_ids: + post = Post.find_one(post_id) + if post is None: + continue + # Get author through relationship (BelongsTo descriptor, needs parentheses to call) + author = post.author() + author_name = author.username if author else "unknown" + + c = Comment( + post_id=post.id, + user_id=bot.id, + body=f"[exp1] Multiprocess sync comment post#{post.id} (author: {author_name}) by pid={os.getpid()}", + is_approved=True, + ) + c.save() + count += 1 + + User.backend().disconnect() + print(f" [sync PID {os.getpid()}] Completed {count} posts") + return count + + +# ───────────────────────────────────────────────────────────────────────────── +# Experiment C: Multiprocess (Asynchronous ActiveRecord) +# ───────────────────────────────────────────────────────────────────────────── + + +async def async_process_post(post_id: int, bot_id: int) -> bool: + """Asynchronously process a single post: query post + get related author + insert comment""" + post = await AsyncPost.find_one(post_id) + if post is None: + return False + # Get author through async relationship (AsyncBelongsTo descriptor, needs parentheses to call) + author = await post.author() + author_name = author.username if author else "unknown" + + c = AsyncComment( + post_id=post.id, + user_id=bot_id, + body=f"[exp1] Multiprocess async comment post#{post.id} (author: {author_name}) by pid={os.getpid()}", + is_approved=True, + ) + await c.save() + return True + + +async def async_worker_main(post_ids: list) -> int: + """ + Async Worker main function: + Call configure() within process, then process post list sequentially with async/await. + Note: Coroutines within a single connection cannot run concurrently—asyncio.gather + would cause conflicts. + Concurrency is achieved across multiple processes: each process holds an independent + PostgreSQL connection. + """ + config = load_config() + # ✅ Async configure() also called in child process (identical method name to sync version, add await) + await AsyncUser.configure(config, AsyncPostgresBackend) + backend = AsyncUser.backend() + AsyncPost.__backend__ = backend + AsyncComment.__backend__ = backend + + bot = await AsyncUser.query().order_by(AsyncUser.c.id).one() + if bot is None: + return 0 + + # Single connection sequential execution: coroutines within same process share same backend + # connection, cannot run concurrently + # Concurrency is achieved across multiple processes with independent connections + count = 0 + for pid in post_ids: + if await async_process_post(pid, bot.id): + count += 1 + + await AsyncUser.backend().disconnect() + print(f" [async PID {os.getpid()}] Completed {count} posts") + return count + + +def worker_async(post_ids: list) -> int: + """Child process entry: Each process independently creates its own event loop""" + return asyncio.run(async_worker_main(post_ids)) + + +# ───────────────────────────────────────────────────────────────────────────── +# Main Program +# ───────────────────────────────────────────────────────────────────────────── + + +def main() -> None: + post_ids = fetch_published_post_ids() + if not post_ids: + print("⚠️ No posts with 'published' status, please run setup_db.py first") + sys.exit(1) + + print(f"Found {len(post_ids)} published posts\n") + chunks = split_chunks(post_ids, NUM_WORKERS) + + # ─── Experiment A: Serial ─── + print("=== Experiment A: Serial Processing (Single Process, Synchronous ActiveRecord) ===") + t0 = time.perf_counter() + serial_count = run_serial(post_ids) + t_serial = time.perf_counter() - t0 + print(f"Serial completed {serial_count} posts, time: {t_serial:.3f}s") + + reset_exp_comments() + + # ─── Experiment B: Multiprocess Sync ─── + print(f"\n=== Experiment B: {NUM_WORKERS} Processes Parallel (Synchronous ActiveRecord) ===") + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_b = pool.map(worker_sync, chunks) + t_sync = time.perf_counter() - t0 + total_b = sum(results_b) + print(f"Parallel completed {total_b} posts, time: {t_sync:.3f}s") + if t_sync > 0: + print(f"Speedup: {t_serial / t_sync:.1f}x (theoretical max {NUM_WORKERS}x)") + + reset_exp_comments() + + # ─── Experiment C: Multiprocess Async ─── + print(f"\n=== Experiment C: {NUM_WORKERS} Processes Parallel (Asynchronous ActiveRecord) ===") + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_c = pool.map(worker_async, chunks) + t_async = time.perf_counter() - t0 + total_c = sum(results_c) + print(f"Parallel completed {total_c} posts, time: {t_async:.3f}s") + if t_async > 0: + print(f"Speedup: {t_serial / t_async:.1f}x (theoretical max {NUM_WORKERS}x)") + + reset_exp_comments() + + print(f"\n{'=' * 55}") + print("Conclusion:") + print(f" Serial (sync): {t_serial:.3f}s") + print(f" Multiprocess sync: {t_sync:.3f}s ({t_serial / t_sync:.1f}x)") + print(f" Multiprocess async: {t_async:.3f}s ({t_serial / t_async:.1f}x)") + print(""" +Key principle: configure() must be called within child processes, applies to both +sync and async versions with identical method names. + +Similarities between PostgreSQL and MySQL: + - configure() establishes real TCP network connections, inheriting connection file + descriptors after fork is more dangerous + - Async backend (psycopg native async) is based on real TCP network I/O + - Both achieve near-linear speedup with multiprocess parallelism + +PostgreSQL-specific advantages: + - RETURNING clause support for INSERT/UPDATE/DELETE returning data + - More refined MVCC implementation, reads don't block writes +""") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py b/docs/examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py new file mode 100644 index 00000000..000a1dc7 --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py @@ -0,0 +1,256 @@ +# exp2_postgres_async_advantage.py - PostgreSQL Async Features and Multiprocess Concurrent Writes +# docs/examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py +""" +Experiment objective: + Explain the single-connection limitation of psycopg async backend + (AsyncPostgresBackend), and how PostgreSQL MVCC enables multiprocess concurrent + writes without extra configuration. + +Experiment has two parts: + + Part 1 — Single-connection limitation (sync vs async within same process) + Within a single process, synchronous serial vs asynchronous sequential (single-connection + limitation: cannot use asyncio.gather for concurrency). + psycopg async backend is single-connection, concurrent access raises RuntimeError. + Conclusion: Under single-connection, async and sync have similar performance; + concurrency advantage is realized across multiple processes. + + Part 2 — Multiprocess concurrent writes (no WAL mode configuration needed) + PostgreSQL MVCC defaults to supporting concurrency, multiple processes writing to + different rows won't block each other. + Contrast with SQLite: default file locking serializes all writes, requires WAL mode + for concurrency. + Conclusion: PostgreSQL supports true concurrent writes out of the box. + +How to run: + python setup_db.py # Initialize database first + python exp2_postgres_async_advantage.py +""" + +from __future__ import annotations + +import asyncio +import multiprocessing +import os +import sys +import time + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import ( # noqa: E402 + AsyncPostgresBackend, + PostgresBackend, +) + +from config_loader import load_config # noqa: E402 +from models import AsyncPost, Post # noqa: E402 + +NUM_WORKERS = 4 + + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + + +def fetch_all_post_ids() -> list: + config = load_config() + Post.configure(config, PostgresBackend) + ids = [p.id for p in Post.query().order_by(Post.c.id).all() if p.id is not None] + Post.backend().disconnect() + return ids + + +def split_chunks(items: list, n: int) -> list: + size = max(1, (len(items) + n - 1) // n) + return [items[i : i + size] for i in range(0, len(items), size)] + + +def reset_posts(all_ids: list) -> None: + config = load_config() + Post.configure(config, PostgresBackend) + for post_id in all_ids: + post = Post.find_one(post_id) + if post: + post.view_count = 0 + post.save() + Post.backend().disconnect() + + +# ───────────────────────────────────────────────────────────────────────────── +# Part 1: Async concurrency within same process (compare with sync serial) +# ───────────────────────────────────────────────────────────────────────────── + + +def run_sync_sequential(post_ids: list) -> float: + """Synchronous serial update of post view_count, with timing.""" + config = load_config() + Post.configure(config, PostgresBackend) + + t0 = time.perf_counter() + for post_id in post_ids: + post = Post.find_one(post_id) + if post: + post.view_count += 1 + post.save() + elapsed = time.perf_counter() - t0 + + Post.backend().disconnect() + return elapsed + + +async def _update_post_async(post_id: int) -> None: + post = await AsyncPost.find_one(post_id) + if post: + post.view_count += 1 + await post.save() + + +async def run_async_concurrent(post_ids: list) -> float: + """Asynchronous sequential update of post view_count, with timing. + + Note: rhosocial-activerecord's single-connection model means coroutines within + the same process cannot access the database concurrently (asyncio.gather would + cause conflicts). + Async advantage is realized across multiple processes: each process holds an + independent connection, enabling true concurrency between processes. + """ + config = load_config() + await AsyncPost.configure(config, AsyncPostgresBackend) + + t0 = time.perf_counter() + # Single connection: coroutines must execute sequentially, cannot run concurrently + for pid in post_ids: + await _update_post_async(pid) + elapsed = time.perf_counter() - t0 + + await AsyncPost.backend().disconnect() + return elapsed + + +# ───────────────────────────────────────────────────────────────────────────── +# Part 2: Multiprocess concurrent writes (MVCC, no WAL config needed) +# ───────────────────────────────────────────────────────────────────────────── + + +def sync_worker(post_ids: list) -> tuple: + """Sync Worker: Update assigned posts, each process has independent connection.""" + config = load_config() + Post.configure(config, PostgresBackend) + + t0 = time.perf_counter() + count = 0 + for post_id in post_ids: + post = Post.find_one(post_id) + if post: + post.view_count += 1 + post.save() + count += 1 + elapsed = time.perf_counter() - t0 + + Post.backend().disconnect() + return count, elapsed + + +async def _async_worker_main(post_ids: list) -> tuple: + config = load_config() + await AsyncPost.configure(config, AsyncPostgresBackend) + # Single connection: coroutines execute sequentially (concurrency is across processes) + t0 = time.perf_counter() + for pid in post_ids: + await _update_post_async(pid) + elapsed = time.perf_counter() - t0 + await AsyncPost.backend().disconnect() + return len(post_ids), elapsed + + +def async_worker(post_ids: list) -> tuple: + return asyncio.run(_async_worker_main(post_ids)) + + +# ───────────────────────────────────────────────────────────────────────────── +# Main Program +# ───────────────────────────────────────────────────────────────────────────── + + +def main() -> None: + all_ids = fetch_all_post_ids() + if not all_ids: + print("⚠️ No post data, please run setup_db.py first") + sys.exit(1) + + print(f"Experiment setup: {NUM_WORKERS} processes, {len(all_ids)} posts total\n") + chunks = split_chunks(all_ids, NUM_WORKERS) + + # ─── Part 1: sync vs async within same process ─── + print("=" * 60) + print("Part 1: Sync serial vs Async sequential within same process (single-connection limitation)") + print("=" * 60) + + reset_posts(all_ids) + t_sync_single = run_sync_sequential(all_ids) + print(f" Sync serial: {len(all_ids)} posts, time: {t_sync_single:.3f}s") + + reset_posts(all_ids) + t_async_single = asyncio.run(run_async_concurrent(all_ids)) + print(f" Async concurrent: {len(all_ids)} posts, time: {t_async_single:.3f}s") + if t_async_single > 0: + ratio = t_sync_single / t_async_single + print(f" Async/Sync ratio: {ratio:.2f}x (similar under single-connection, concurrency advantage across processes)") + + # ─── Part 2: Multiprocess concurrent writes ─── + print(f"\n{'=' * 60}") + print("Part 2: Multiprocess concurrent writes (PostgreSQL MVCC, no WAL config needed)") + print("=" * 60) + + reset_posts(all_ids) + print(f"\n Sync multiprocess ({NUM_WORKERS} processes):") + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_sync = pool.map(sync_worker, chunks) + t_mp_sync = time.perf_counter() - t0 + total_sync = sum(r[0] for r in results_sync) + print(f" Processed {total_sync} posts, time: {t_mp_sync:.3f}s") + + reset_posts(all_ids) + print(f"\n Async multiprocess ({NUM_WORKERS} processes, asyncio sequential within each process):") + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_async = pool.map(async_worker, chunks) + t_mp_async = time.perf_counter() - t0 + total_async = sum(r[0] for r in results_async) + print(f" Processed {total_async} posts, time: {t_mp_async:.3f}s") + + reset_posts(all_ids) + + print(f"\n{'=' * 60}") + print("Summary:") + print(f" Part 1 — Same process sync serial: {t_sync_single:.3f}s async sequential: {t_async_single:.3f}s") + print(f" Part 2 — Multiprocess sync: {t_mp_sync:.3f}s Multiprocess async: {t_mp_async:.3f}s") + print(""" +Conclusion: + rhosocial-activerecord uses a single-connection model (one ActiveRecord class bound to one connection). + Single-connection limitation: Coroutines within same process cannot access database concurrently + —asyncio.gather would cause conflicts. + Concurrency is achieved across multiple processes: each process holds independent PostgreSQL TCP connection, + enabling true concurrency between processes. + + PostgreSQL multiprocess concurrent writes require no extra configuration (no WAL, no journal_mode), + MVCC by default allows multiple processes to write different rows without blocking each other. + + psycopg async advantages: + - await doesn't block event loop, can handle other tasks while waiting for I/O + - Multiprocess + async combination is more efficient than pure multiprocess sync under high-latency networks + - Note: Connection pooling is needed for true single-process concurrent queries (this project uses single-connection model) + +PostgreSQL vs MySQL: + - Both support native async I/O (unlike SQLite using thread pool simulation) + - PostgreSQL uses MVCC, MySQL uses row-level locking + - Both support concurrent writes without special configuration +""") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py b/docs/examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py new file mode 100644 index 00000000..bcdb6660 --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py @@ -0,0 +1,173 @@ +# exp3_deadlock_wrong.py - Row Lock Order Conflict Causing Deadlock (Anti-pattern) +# docs/examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py +""" +Experiment objective: + Demonstrate that when multiprocess concurrent writes have inconsistent lock ordering, + PostgreSQL detects deadlock and forcibly rolls back one of the transactions (raises + OperationalError/DatabaseError, SQLSTATE=40P01). + +Deadlock scenario: + Two groups of Workers update the same set of posts in opposite order: + - Worker group A (even Workers): Lock rows with smaller IDs first, then larger IDs + - Worker group B (odd Workers): Lock rows with larger IDs first, then smaller IDs + + When A holds lock on row 1 and waits for row 2, while B holds lock on row 2 and + waits for row 1, PostgreSQL automatically detects the deadlock and rolls back + one transaction, allowing the other to proceed. + +Note: + - This is an anti-pattern demonstrating unsafe lock ordering + - PostgreSQL deadlock detection is automatic, the rolled-back process receives OperationalError + - Correct solution see exp4_partition_correct.py: Always lock resources in ascending order + +How to run: + python setup_db.py # Initialize database first + python exp3_deadlock_wrong.py +""" + +from __future__ import annotations + +import multiprocessing +import os +import sys +import time + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import PostgresBackend # noqa: E402 + +from config_loader import load_config # noqa: E402 +from models import Post # noqa: E402 + +NUM_WORKERS = 4 +# Number of post IDs each Worker competes for (intentionally overlapping to create deadlock opportunity) +BATCH_IDS = [1, 2, 3, 4, 5] # All Workers try to update these rows + + +def worker_wrong_order(args: tuple) -> dict: + """ + ❌ Anti-pattern: Worker decides lock order based on worker_id parity. + Even Workers use ascending order (1→2→3…), odd Workers use descending order (…3→2→1). + Inconsistent lock ordering makes deadlock highly likely with concurrent Workers. + """ + worker_id, post_ids = args + config = load_config() + Post.configure(config, PostgresBackend) + Post.__backend__ = Post.backend() + + # Odd Workers reverse order to create lock order conflict with even Workers + if worker_id % 2 == 1: + ordered_ids = list(reversed(post_ids)) + else: + ordered_ids = list(post_ids) + + result = { + "worker_id": worker_id, + "success": [], + "deadlock": False, + "error": None, + } + + try: + with Post.transaction(): + for post_id in ordered_ids: + post = Post.find_one(post_id) + if post is None: + continue + # Simulate processing time to increase deadlock window + time.sleep(0.01) + post.view_count += 1 + post.save() + result["success"].append(post_id) + except Exception as e: + err_str = str(e) + # PostgreSQL deadlock error code is SQLSTATE 40P01 + # psycopg 3's exception object may have sqlstate attribute + is_deadlock = False + if hasattr(e, 'sqlstate') and e.sqlstate == '40P01': + is_deadlock = True + elif '40P01' in err_str or 'deadlock' in err_str.lower(): + is_deadlock = True + + if is_deadlock: + result["deadlock"] = True + result["error"] = f"Deadlock (PostgreSQL SQLSTATE 40P01): {err_str[:80]}" + else: + result["error"] = f"Other error: {err_str[:80]}" + finally: + try: + Post.backend().disconnect() + except Exception: + pass + + status = "💀 Deadlock rollback" if result["deadlock"] else ("✅ Success" if not result["error"] else "❌ Error") + print(f" Worker {worker_id} ({status}): order={ordered_ids[:3]}…, succeeded={len(result['success'])} posts") + if result["error"]: + print(f" → {result['error']}") + return result + + +def verify_results() -> dict: + """Count final results: view_count > 0 indicates successfully processed.""" + config = load_config() + Post.configure(config, PostgresBackend) + posts = Post.query().where(Post.c.id.in_(BATCH_IDS)).all() + stats = {p.id: p.view_count for p in posts} + Post.backend().disconnect() + return stats + + +def main() -> None: + print("=== Anti-pattern: Inconsistent Lock Order (PostgreSQL Deadlock Demo) ===\n") + print(f"All {NUM_WORKERS} Workers compete for the same post IDs: {BATCH_IDS}") + print("Even Workers lock in ascending order, odd Workers in descending order, creating deadlock conditions\n") + + args = [(i, BATCH_IDS) for i in range(NUM_WORKERS)] + + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results = pool.map(worker_wrong_order, args) + elapsed = time.perf_counter() - t0 + + # Statistics + deadlock_count = sum(1 for r in results if r["deadlock"]) + success_count = sum(1 for r in results if not r["error"]) + error_count = sum(1 for r in results if r["error"] and not r["deadlock"]) + + print(f"\nTime elapsed: {elapsed:.3f}s") + print(f"Results: {success_count} Workers succeeded, {deadlock_count} deadlocked, {error_count} other errors") + + if deadlock_count > 0: + print(f"\n⚠️ Found {deadlock_count} Workers rolled back due to deadlock by PostgreSQL!") + print(" All operations of rolled-back Workers did not take effect (transaction atomicity guarantee)") + else: + print("\n No deadlock triggered in this run (timing factor, run multiple times to reproduce)") + + view_counts = verify_results() + print("\nFinal view_count (ideal value should be 1, but some Workers rolled back causing possible 0):") + for pid, vc in sorted(view_counts.items()): + mark = "✅" if vc > 0 else "❌" + print(f" Post #{pid}: view_count={vc} {mark}") + + print(""" +Conclusion (anti-pattern): + - PostgreSQL automatically detects deadlocks (deadlock detection algorithm, enabled by default) + - When deadlock occurs, PostgreSQL chooses the lower-cost transaction to roll back, allowing the other to proceed + - Rolled-back Worker receives OperationalError (SQLSTATE=40P01) + - If this exception is not caught and retried, all work from the rolled-back Worker is lost + +Correct solutions: + 1. Always lock resources in fixed order (e.g., primary key ascending) → see exp4 Solution A + 2. Catch deadlock exception and retry → doesn't rely on order, but requires retry logic + 3. Data partitioning, each Worker processes non-overlapping datasets → see exp4 Solution B + +PostgreSQL vs MySQL deadlock error codes: + - PostgreSQL: SQLSTATE 40P01 + - MySQL: errno 1213 +""") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py b/docs/examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py new file mode 100644 index 00000000..2dbe53e8 --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py @@ -0,0 +1,412 @@ +# exp4_partition_correct.py - Data Partitioning + Atomic Claiming Correct Implementation (PostgreSQL Version) +# docs/examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py +""" +Experiment objective: + Demonstrate two correct multiprocess parallel processing strategies using real PostgreSQL ActiveRecord ORM. + Both synchronous and asynchronous versions are provided with identical method names. + Async version only requires adding await. + + Solution A: Data Partitioning + Distribute posts to Workers by user_id, no overlap, no contention. + Demonstrates relationships: Worker processes posts for assigned users, traverses their comments. + + Solution B: Atomic Claiming (query + update within transaction) + Query pending posts and immediately update status within transaction. + Transaction isolation guarantees no duplicates. + PostgreSQL MVCC makes this pattern more efficient than SQLite (no file-level lock contention). + + Solution C: Atomic Claiming + Deadlock Retry + Built on Solution B, catch PostgreSQL deadlock exception (SQLSTATE 40P01) and automatically retry. + Recommended for production: doesn't rely on partitionable data, balances safety and robustness. + +Verification: Each post is processed exactly once (view_count > 0). + +How to run: + python setup_db.py # Initialize database first + python exp4_partition_correct.py +""" + +from __future__ import annotations + +import asyncio +import multiprocessing +import os +import sys +import time + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import ( # noqa: E402 + AsyncPostgresBackend, + PostgresBackend, +) + +from config_loader import load_config # noqa: E402 +from models import AsyncComment, AsyncPost, AsyncUser, Comment, Post, User # noqa: E402 + +NUM_WORKERS = 4 +CLAIM_BATCH = 3 # Batch size for atomic claiming +MAX_RETRY = 3 # Maximum deadlock retry attempts + + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + + +def reset_all_posts() -> None: + config = load_config() + Post.configure(config, PostgresBackend) + posts = Post.query().all() + for p in posts: + p.status = "draft" + p.view_count = 0 + p.save() + Post.backend().disconnect() + + +def fetch_user_ids() -> list: + config = load_config() + User.configure(config, PostgresBackend) + ids = [u.id for u in User.query().order_by(User.c.id).all() if u.id is not None] + User.backend().disconnect() + return ids + + +def verify_no_duplicates() -> tuple: + """Verify all posts are published and view_count > 0 (processed exactly once)""" + config = load_config() + Post.configure(config, PostgresBackend) + posts = Post.query().where(Post.c.status == "published").all() + ok = all(p.view_count > 0 for p in posts) + count = len(posts) + Post.backend().disconnect() + return ok, count + + +def _is_deadlock(exc: Exception) -> bool: + """Check if exception is PostgreSQL deadlock (SQLSTATE 40P01)""" + # psycopg 3's error object has sqlstate attribute + if hasattr(exc, 'sqlstate'): + return exc.sqlstate == '40P01' + msg = str(exc) + return '40P01' in msg or 'deadlock' in msg.lower() + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Solution A: Data Partitioning (Synchronous Version) +# ═══════════════════════════════════════════════════════════════════════════════ + + +def worker_partition_sync(user_ids: list) -> int: + """ + Sync Worker: Only process posts for assigned user_ids, traverse comments via relationships. + + Demonstrates relationships: + - User --(has_many)--> Post --(has_many)--> Comment + - Count approved comments per post, write to view_count + """ + config = load_config() + User.configure(config, PostgresBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + count = 0 + for uid in user_ids: + posts = Post.query().where(Post.c.user_id == uid).all() + for post in posts: + # Get comments through HasMany relationship (needs parentheses to call) + approved = [c for c in post.comments() if c.is_approved] + post.status = "published" + post.view_count = 1 + len(approved) + post.save() + count += 1 + + User.backend().disconnect() + print(f" [Partition sync PID {os.getpid()}] Processed {count} posts") + return count + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Solution A: Data Partitioning (Asynchronous Version) — identical method names, add await +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def _async_partition_main(user_ids: list) -> int: + config = load_config() + await AsyncUser.configure(config, AsyncPostgresBackend) + backend = AsyncUser.backend() + AsyncPost.__backend__ = backend + AsyncComment.__backend__ = backend + + count = 0 + for uid in user_ids: + posts = await AsyncPost.query().where(AsyncPost.c.user_id == uid).all() + for post in posts: + # Get comments through async HasMany relationship (add await and parentheses) + approved = [c for c in await post.comments() if c.is_approved] + post.status = "published" + post.view_count = 1 + len(approved) + await post.save() + count += 1 + + await AsyncUser.backend().disconnect() + print(f" [Partition async PID {os.getpid()}] Processed {count} posts") + return count + + +def worker_partition_async(user_ids: list) -> int: + return asyncio.run(_async_partition_main(user_ids)) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Solution B: Atomic Claiming (Synchronous Version) +# ═══════════════════════════════════════════════════════════════════════════════ + + +def claim_posts_sync(batch: int = CLAIM_BATCH) -> list: + """ + Atomically claim within transaction: query + update completed in same transaction. + PostgreSQL MVCC ensures only one Worker can successfully claim the same batch. + """ + with Post.transaction(): + pending = Post.query().where(Post.c.status == "draft").order_by(Post.c.id).limit(batch).all() + if not pending: + return [] + for post in pending: + post.status = "processing" + post.view_count = 0 + post.save() + return pending + + +def worker_atomic_sync(worker_id: int) -> int: + """Sync Worker: Loop atomic claiming until no tasks""" + config = load_config() + User.configure(config, PostgresBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + total = 0 + while True: + batch = claim_posts_sync() + if not batch: + break + for post in batch: + approved_count = len([c for c in post.comments() if c.is_approved]) + post.status = "published" + post.view_count = 1 + approved_count + post.save() + total += 1 + + User.backend().disconnect() + print(f" [Atomic sync Worker {worker_id}] Processed {total} posts") + return total + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Solution B: Atomic Claiming (Asynchronous Version) — identical method names, add await +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def claim_posts_async(batch: int = CLAIM_BATCH) -> list: + """Async atomic claiming: async with transaction(), structure completely symmetric with sync version""" + async with AsyncPost.transaction(): + pending = await ( + AsyncPost.query().where(AsyncPost.c.status == "draft").order_by(AsyncPost.c.id).limit(batch).all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + post.view_count = 0 + await post.save() + return pending + + +async def _async_atomic_main(worker_id: int) -> int: + config = load_config() + await AsyncUser.configure(config, AsyncPostgresBackend) + backend = AsyncUser.backend() + AsyncPost.__backend__ = backend + AsyncComment.__backend__ = backend + + total = 0 + while True: + batch = await claim_posts_async() + if not batch: + break + for post in batch: + approved_count = len([c for c in await post.comments() if c.is_approved]) + post.status = "published" + post.view_count = 1 + approved_count + await post.save() + total += 1 + + await AsyncUser.backend().disconnect() + print(f" [Atomic async Worker {worker_id}] Processed {total} posts") + return total + + +def worker_atomic_async(worker_id: int) -> int: + return asyncio.run(_async_atomic_main(worker_id)) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Solution C: Atomic Claiming + Deadlock Retry (PostgreSQL Production Recommended) +# ═══════════════════════════════════════════════════════════════════════════════ + + +def claim_posts_with_retry(batch: int = CLAIM_BATCH, max_retry: int = MAX_RETRY) -> list: + """Atomic claiming with PostgreSQL deadlock exception (SQLSTATE 40P01) catch and retry.""" + for attempt in range(max_retry): + try: + with Post.transaction(): + pending = Post.query().where(Post.c.status == "draft").order_by(Post.c.id).limit(batch).all() + if not pending: + return [] + for post in pending: + post.status = "processing" + post.view_count = 0 + post.save() + return pending + except Exception as e: + if _is_deadlock(e) and attempt < max_retry - 1: + time.sleep(0.05 * (attempt + 1)) # Exponential backoff + continue + raise + return [] + + +def worker_atomic_retry(worker_id: int) -> int: + """Sync Worker (with deadlock retry): PostgreSQL production recommended solution""" + config = load_config() + User.configure(config, PostgresBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + total = 0 + while True: + batch = claim_posts_with_retry() + if not batch: + break + for post in batch: + approved_count = len([c for c in post.comments() if c.is_approved]) + post.status = "published" + post.view_count = 1 + approved_count + post.save() + total += 1 + + User.backend().disconnect() + print(f" [Retry sync Worker {worker_id}] Processed {total} posts") + return total + + +# ───────────────────────────────────────────────────────────────────────────── +# Main Program +# ───────────────────────────────────────────────────────────────────────────── + + +def main() -> None: + user_ids = fetch_user_ids() + if not user_ids: + print("⚠️ No user data, please run setup_db.py first") + sys.exit(1) + + size = max(1, (len(user_ids) + NUM_WORKERS - 1) // NUM_WORKERS) + uid_chunks = [user_ids[i : i + size] for i in range(0, len(user_ids), size)] + + # ─── Solution A: Data Partitioning (Sync) ─── + print("=== Solution A: Data Partitioning (Synchronous ActiveRecord) ===") + reset_all_posts() + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_a_sync = pool.map(worker_partition_sync, uid_chunks) + t_a_sync = time.perf_counter() - t0 + ok_a_sync, done_a_sync = verify_no_duplicates() + print( + f" Result: Workers processed {sum(results_a_sync)} posts, published {done_a_sync} posts, " + f"time {t_a_sync:.3f}s {'✅ No duplicates' if ok_a_sync else '❌ Has duplicates'}" + ) + + # ─── Solution A: Data Partitioning (Async) ─── + print("\n=== Solution A: Data Partitioning (Asynchronous ActiveRecord) ===") + reset_all_posts() + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_a_async = pool.map(worker_partition_async, uid_chunks) + t_a_async = time.perf_counter() - t0 + ok_a_async, done_a_async = verify_no_duplicates() + print( + f" Result: Workers processed {sum(results_a_async)} posts, published {done_a_async} posts, " + f"time {t_a_async:.3f}s {'✅ No duplicates' if ok_a_async else '❌ Has duplicates'}" + ) + + # ─── Solution B: Atomic Claiming (Sync) ─── + print("\n=== Solution B: Atomic Claiming (Synchronous ActiveRecord) ===") + reset_all_posts() + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_b_sync = pool.map(worker_atomic_sync, list(range(NUM_WORKERS))) + t_b_sync = time.perf_counter() - t0 + ok_b_sync, done_b_sync = verify_no_duplicates() + print( + f" Result: Workers processed {sum(results_b_sync)} posts, published {done_b_sync} posts, " + f"time {t_b_sync:.3f}s {'✅ No duplicates' if ok_b_sync else '❌ Has duplicates'}" + ) + + # ─── Solution B: Atomic Claiming (Async) ─── + print("\n=== Solution B: Atomic Claiming (Asynchronous ActiveRecord) ===") + reset_all_posts() + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_b_async = pool.map(worker_atomic_async, list(range(NUM_WORKERS))) + t_b_async = time.perf_counter() - t0 + ok_b_async, done_b_async = verify_no_duplicates() + print( + f" Result: Workers processed {sum(results_b_async)} posts, published {done_b_async} posts, " + f"time {t_b_async:.3f}s {'✅ No duplicates' if ok_b_async else '❌ Has duplicates'}" + ) + + # ─── Solution C: Atomic Claiming + Deadlock Retry (Sync) ─── + print("\n=== Solution C: Atomic Claiming + Deadlock Retry (PostgreSQL Production Recommended, Sync) ===") + reset_all_posts() + t0 = time.perf_counter() + with multiprocessing.Pool(processes=NUM_WORKERS) as pool: + results_c = pool.map(worker_atomic_retry, list(range(NUM_WORKERS))) + t_c = time.perf_counter() - t0 + ok_c, done_c = verify_no_duplicates() + print( + f" Result: Workers processed {sum(results_c)} posts, published {done_c} posts, " + f"time {t_c:.3f}s {'✅ No duplicates' if ok_c else '❌ Has duplicates'}" + ) + + reset_all_posts() + + all_ok = ok_a_sync and ok_a_async and ok_b_sync and ok_b_async and ok_c + print(f"\n{'=' * 60}") + print("Solution comparison:") + print(f" A. Data partitioning Sync: {t_a_sync:.3f}s Async: {t_a_async:.3f}s") + print(f" B. Atomic claiming Sync: {t_b_sync:.3f}s Async: {t_b_async:.3f}s") + print(f" C. Atomic + retry Sync: {t_c:.3f}s") + print(f"\nOverall verification: {'✅ All solutions have no duplicate processing' if all_ok else '❌ Issues found'}") + print(""" +PostgreSQL-specific optimization notes: + - Solution A (Data Partitioning): Each Worker operates on different rows, MVCC has no lock + contention, highest efficiency + - Solution B (Atomic Claiming): PostgreSQL MVCC provides precise isolation, performs better than SQLite + - Solution C (Deadlock Retry): Production recommended, doesn't rely on partitionable data, + automatically handles deadlock situations + - Compared to SQLite WAL mode, PostgreSQL supports concurrent writes without extra configuration + +PostgreSQL vs MySQL deadlock handling: + - PostgreSQL deadlock error code: SQLSTATE 40P01 + - MySQL deadlock error code: errno 1213 + - Both have automatic deadlock detection and retry mechanisms +""") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py b/docs/examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py new file mode 100644 index 00000000..25f82b2a --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py @@ -0,0 +1,189 @@ +# exp5_multithread_warning.py - Issues with Multithreaded PostgreSQL Connection Sharing (Anti-pattern) +# docs/examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py +""" +Experiment objective: + Demonstrate the danger of multithreaded PostgreSQL connection sharing, and that + per-thread configure() doesn't solve the problem. + +Key differences from SQLite version: + SQLite has check_same_thread=True protection, cross-thread access directly raises error. + PostgreSQL (psycopg) doesn't have this protection, + but connection objects are also not thread-safe—concurrent operations cause cursor state corruption. + +Scenario descriptions: + Scenario 1: Shared __backend__ (dangerous) + Multiple threads share the same PostgresBackend instance, concurrent queries/writes cause cursor corruption. + psycopg officially states connections don't support multi-threaded concurrent use. + + Scenario 2: Each thread calls configure() (❌ Doesn't work) + configure() writes to class attribute __backend__, later ones overwrite earlier ones. + All threads actually share the last configure()'s created backend instance. + Demonstrate by collecting id(Post.backend()) from each thread to prove instances aren't independent. + +Conclusion: + Multithreading cannot solve connection sharing problem, correct solution is multiprocessing + (each process has independent connection). + +How to run: + python setup_db.py # Initialize database first + python exp5_multithread_warning.py +""" + +from __future__ import annotations + +import sys +import os +import threading +import time +from typing import List + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import PostgresBackend # noqa: E402 + +from config_loader import load_config # noqa: E402 +from models import Post # noqa: E402 + +NUM_THREADS = 4 + + +# ───────────────────────────────────────────────────────────────────────────── +# Scenario 1: Shared __backend__ (dangerous, possible cursor corruption) +# ───────────────────────────────────────────────────────────────────────────── + + +def scenario_1_shared_backend() -> None: + """❌ Dangerous: Multiple threads share the same PostgresBackend instance.""" + print("=== Scenario 1: Shared __backend__ (❌ Dangerous) ===") + + # Initialize in main thread (analogous to parent process configure then fork) + config = load_config() + Post.configure(config, PostgresBackend) + shared_backend_id = id(Post.backend()) + print(f" Main thread backend id: {shared_backend_id}") + + errors: List[str] = [] + lock = threading.Lock() + + def thread_worker(thread_id: int) -> None: + # All threads share the same backend, concurrent operations may cause cursor state corruption + try: + # Two threads execute queries simultaneously, cursor may be overwritten by another thread + posts = Post.query().where(Post.c.id > 0).limit(3).all() + time.sleep(0.01) # Increase concurrency window + # Query again, check if results are consistent + posts2 = Post.query().where(Post.c.id > 0).limit(3).all() + if len(posts) != len(posts2): + with lock: + errors.append(f"Thread {thread_id}: Inconsistent query results ({len(posts)} vs {len(posts2)})") + except Exception as e: + with lock: + errors.append(f"Thread {thread_id} exception: {e!s:.80}") + + threads = [threading.Thread(target=thread_worker, args=(i,)) for i in range(NUM_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + if errors: + print(f" ⚠️ Found {len(errors)} errors or inconsistencies (proves multithreaded connection sharing is unsafe):") + for err in errors[:3]: + print(f" - {err}") + else: + print(" No errors in this run (timing factor, run multiple times to reproduce cursor corruption)") + print(" Note: psycopg explicitly states connections don't support multi-threaded concurrent use") + + try: + Post.backend().disconnect() + except Exception: + pass + print() + + +# ───────────────────────────────────────────────────────────────────────────── +# Scenario 2: Each thread calls configure() (❌ Doesn't work, class attribute overwritten) +# ───────────────────────────────────────────────────────────────────────────── + + +def scenario_2_per_thread_configure() -> None: + """❌ Doesn't work: Each thread calls configure(), but __backend__ is a class attribute, later overwrites earlier.""" + print("=== Scenario 2: Each thread calls configure() (❌ Doesn't work, class attributes overwrite each other) ===") + + backend_ids: List[int] = [] + lock = threading.Lock() + + def thread_worker(thread_id: int) -> None: + config = load_config() + # Each thread calls configure(), appears to be independent + Post.configure(config, PostgresBackend) + time.sleep(0.02) # Wait for all threads to complete configure + # Collect backend instance id + with lock: + backend_ids.append(id(Post.backend())) + + threads = [threading.Thread(target=thread_worker, args=(i,)) for i in range(NUM_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + unique_ids = set(backend_ids) + print(f" {NUM_THREADS} threads each called configure(), actual backend instance count: {len(unique_ids)}") + print(f" (Expected {NUM_THREADS}, got {len(unique_ids)})") + if len(unique_ids) == 1: + print(" ❌ All threads share the same backend instance — class attribute __backend__ overwritten by last configure()") + else: + print(" ⚠️ Multiple instances exist (timing factor), but still unstable, should not rely on this behavior") + + try: + Post.backend().disconnect() + except Exception: + pass + print() + + +# ───────────────────────────────────────────────────────────────────────────── +# Main Program +# ───────────────────────────────────────────────────────────────────────────── + + +def main() -> None: + print("PostgreSQL Multithreaded Connection Sharing Issue Demo (Anti-pattern)") + print("Note: PostgreSQL doesn't have SQLite's check_same_thread protection, but also doesn't support concurrent connection sharing\n") + + scenario_1_shared_backend() + scenario_2_per_thread_configure() + + print("=" * 60) + print("Conclusion:") + print(""" + psycopg official documentation clearly states: + "Connections are not thread-safe: you cannot share a connection + between threads. Each thread must use its own connection." + + Multithreading + shared __backend__ is undefined behavior, may cause: + - Cursor state corruption (one thread's query results overwritten by another) + - Transaction boundary confusion (BEGIN/COMMIT order mixed) + - Connection state corruption (auth info, encoding settings overwritten) + + Per-thread configure() doesn't solve the problem: + - __backend__ is a class attribute, all threads share the same class, later overwrites earlier + - Even if each configure() succeeds, other threads immediately use the same backend + + Correct solution: Multiprocessing + - Each process has independent memory space, __backend__ not shared + - Each process calls configure() to establish independent PostgreSQL TCP connection + - Processes coordinate task distribution via multiprocessing.Pool / Queue + - See exp1_basic_multiprocess.py and exp4_partition_correct.py + +PostgreSQL vs SQLite multithreading protection: + - SQLite: check_same_thread=True default protection, cross-thread direct error + - PostgreSQL: No built-in protection, but officially not supported, undefined behavior +""") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/models.py b/docs/examples/chapter_08_scenarios/parallel_workers/models.py new file mode 100644 index 00000000..cdd42a0a --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/models.py @@ -0,0 +1,192 @@ +# models.py - Shared Model Definitions for Parallel Worker Experiments (PostgreSQL Version) +# docs/examples/chapter_08_scenarios/parallel_workers/models.py +""" +Model hierarchy: + User --(has_many)--> Post --(has_many)--> Comment + User <--(belongs_to)-- Post + Post <--(belongs_to)-- Comment + User <--(belongs_to)-- Comment (comment author) + +Each model provides both synchronous (inheriting ActiveRecord) and asynchronous +(inheriting AsyncActiveRecord) versions with identical method names. +Async version only requires adding await. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import ClassVar, Optional + +from rhosocial.activerecord.base.field_proxy import FieldProxy +from rhosocial.activerecord.field import IntegerPKMixin, TimestampMixin +from rhosocial.activerecord.model import ActiveRecord, AsyncActiveRecord +from rhosocial.activerecord.relation import BelongsTo, HasMany +from rhosocial.activerecord.relation.async_descriptors import ( + AsyncBelongsTo, + AsyncHasMany, +) + + +# ───────────────────────────────────────── +# Synchronous Models +# ───────────────────────────────────────── + + +class User(IntegerPKMixin, TimestampMixin, ActiveRecord): + """Blog user (synchronous)""" + + __table_name__ = "users" + + id: Optional[int] = None + username: str + email: str + is_active: bool = True + + c: ClassVar[FieldProxy] = FieldProxy() + + posts: ClassVar[HasMany[Post]] = HasMany(foreign_key="user_id", inverse_of="author") + comments: ClassVar[HasMany[Comment]] = HasMany(foreign_key="user_id", inverse_of="author") + + +class Post(IntegerPKMixin, TimestampMixin, ActiveRecord): + """Blog post (synchronous)""" + + __table_name__ = "posts" + + id: Optional[int] = None + user_id: int + title: str + body: str + status: str = "draft" # draft / processing / published + view_count: int = 0 + + c: ClassVar[FieldProxy] = FieldProxy() + + author: ClassVar[BelongsTo[User]] = BelongsTo(foreign_key="user_id", inverse_of="posts") + comments: ClassVar[HasMany[Comment]] = HasMany(foreign_key="post_id", inverse_of="post") + + +class Comment(IntegerPKMixin, TimestampMixin, ActiveRecord): + """Comment (synchronous)""" + + __table_name__ = "comments" + + id: Optional[int] = None + post_id: int + user_id: int + body: str + is_approved: bool = False + + c: ClassVar[FieldProxy] = FieldProxy() + + post: ClassVar[BelongsTo[Post]] = BelongsTo(foreign_key="post_id", inverse_of="comments") + author: ClassVar[BelongsTo[User]] = BelongsTo(foreign_key="user_id", inverse_of="comments") + + +# ───────────────────────────────────────── +# Asynchronous Models (identical method names to sync version, add await) +# ───────────────────────────────────────── + + +class AsyncUser(IntegerPKMixin, TimestampMixin, AsyncActiveRecord): + """Blog user (asynchronous)""" + + __table_name__ = "users" + + id: Optional[int] = None + username: str + email: str + is_active: bool = True + + c: ClassVar[FieldProxy] = FieldProxy() + + posts: ClassVar[AsyncHasMany[AsyncPost]] = AsyncHasMany(foreign_key="user_id", inverse_of="author") + comments: ClassVar[AsyncHasMany[AsyncComment]] = AsyncHasMany(foreign_key="user_id", inverse_of="author") + + +class AsyncPost(IntegerPKMixin, TimestampMixin, AsyncActiveRecord): + """Blog post (asynchronous)""" + + __table_name__ = "posts" + + id: Optional[int] = None + user_id: int + title: str + body: str + status: str = "draft" + view_count: int = 0 + + c: ClassVar[FieldProxy] = FieldProxy() + + author: ClassVar[AsyncBelongsTo[AsyncUser]] = AsyncBelongsTo(foreign_key="user_id", inverse_of="posts") + comments: ClassVar[AsyncHasMany[AsyncComment]] = AsyncHasMany(foreign_key="post_id", inverse_of="post") + + +class AsyncComment(IntegerPKMixin, TimestampMixin, AsyncActiveRecord): + """Comment (asynchronous)""" + + __table_name__ = "comments" + + id: Optional[int] = None + post_id: int + user_id: int + body: str + is_approved: bool = False + + c: ClassVar[FieldProxy] = FieldProxy() + + post: ClassVar[AsyncBelongsTo[AsyncPost]] = AsyncBelongsTo(foreign_key="post_id", inverse_of="comments") + author: ClassVar[AsyncBelongsTo[AsyncUser]] = AsyncBelongsTo(foreign_key="user_id", inverse_of="comments") + + +# ───────────────────────────────────────── +# Table Creation DDL (PostgreSQL) +# ───────────────────────────────────────── + +SCHEMA_SQL = """\ +DROP TABLE IF EXISTS comments CASCADE; +DROP TABLE IF EXISTS posts CASCADE; +DROP TABLE IF EXISTS users CASCADE; + +CREATE TABLE users ( + id SERIAL NOT NULL PRIMARY KEY, + username VARCHAR(64) NOT NULL, + email VARCHAR(255) NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP WITHOUT TIME ZONE NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NULL, + CONSTRAINT uq_username UNIQUE (username) +); + +CREATE TABLE posts ( + id SERIAL NOT NULL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + title VARCHAR(255) NOT NULL, + body TEXT NOT NULL DEFAULT '', + status VARCHAR(20) NOT NULL DEFAULT 'draft', + view_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP WITHOUT TIME ZONE NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NULL +); + +CREATE INDEX idx_posts_user_id ON posts(user_id); +CREATE INDEX idx_posts_status ON posts(status); + +CREATE TABLE comments ( + id SERIAL NOT NULL PRIMARY KEY, + post_id INTEGER NOT NULL REFERENCES posts(id), + user_id INTEGER NOT NULL REFERENCES users(id), + body TEXT NOT NULL, + is_approved BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP WITHOUT TIME ZONE NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NULL +); + +CREATE INDEX idx_comments_post_id ON comments(post_id); +CREATE INDEX idx_comments_user_id ON comments(user_id); +""" + + +def now_utc() -> datetime: + """Return current UTC time (naive datetime, PostgreSQL friendly)""" + return datetime.now(timezone.utc).replace(tzinfo=None) diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/setup_db.py b/docs/examples/chapter_08_scenarios/parallel_workers/setup_db.py new file mode 100644 index 00000000..52b02048 --- /dev/null +++ b/docs/examples/chapter_08_scenarios/parallel_workers/setup_db.py @@ -0,0 +1,202 @@ +# setup_db.py - PostgreSQL Parallel Worker Experiment Database Initialization Script +# docs/examples/chapter_08_scenarios/parallel_workers/setup_db.py +""" +Initialize test database (drop old tables → create tables → insert test data). + +Usage: + python setup_db.py # Synchronous initialization (default) + python setup_db.py --async # Asynchronous initialization (same effect) + python setup_db.py --scenario postgres_16 # Specify YAML scenario + +Note: + - Run this script before each experiment to reset the database + - Drops and recreates users / posts / comments tables +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys + +_src = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "src")) +if _src not in sys.path: + sys.path.insert(0, _src) + +from rhosocial.activerecord.backend.impl.postgres import ( # noqa: E402 + AsyncPostgresBackend, + PostgresBackend, +) + +from config_loader import load_config, show_active_config # noqa: E402 +from models import ( # noqa: E402 + AsyncComment, + AsyncPost, + AsyncUser, + Comment, + Post, + SCHEMA_SQL, + User, +) + +NUM_USERS = 5 +NUM_POSTS_PER_USER = 4 # Total 20 posts +NUM_COMMENTS_PER_POST = 3 # Total 60 comments + + +# ───────────────────────────────────────────────────────────────────────────── +# Synchronous Initialization +# ───────────────────────────────────────────────────────────────────────────── + + +def init_sync(scenario: str = None) -> None: + """Initialize database synchronously and insert test data.""" + config = load_config(scenario) + print("=== Synchronous Initialization Mode ===") + print("Connection info:") + show_active_config(config) + print() + + User.configure(config, PostgresBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + # Create tables (PostgreSQL supports multi-statement execution) + print("Dropping old tables and creating new ones...") + backend = User.backend() + backend.execute(SCHEMA_SQL) + print("Tables created, inserting test data...") + + users = [] + for i in range(1, NUM_USERS + 1): + u = User( + username=f"user{i:02d}", + email=f"user{i:02d}@example.com", + is_active=True, + ) + u.save() + users.append(u) + print(f" User #{u.id}: {u.username}") + + post_count = 0 + comment_count = 0 + for u in users: + for j in range(1, NUM_POSTS_PER_USER + 1): + p = Post( + user_id=u.id, + title=f"{u.username}'s post #{j}", + body=f"This is the body of {u.username}'s post #{j}.", + status="published", + view_count=0, + ) + p.save() + post_count += 1 + for k in range(1, NUM_COMMENTS_PER_POST + 1): + c = Comment( + post_id=p.id, + user_id=users[(k - 1) % len(users)].id, + body=f"Comment #{k} on post #{p.id}", + is_approved=(k % 2 == 1), # Odd comments are approved + ) + c.save() + comment_count += 1 + + print(f"\n Inserted {post_count} posts") + print(f" Inserted {comment_count} comments") + + User.backend().disconnect() + print("\n✓ Synchronous initialization complete") + print(f" Users {NUM_USERS} / Posts {post_count} / Comments {comment_count}") + + +# ───────────────────────────────────────────────────────────────────────────── +# Asynchronous Initialization +# ───────────────────────────────────────────────────────────────────────────── + + +async def init_async(scenario: str = None) -> None: + """Initialize database asynchronously and insert test data (identical method name to sync version, add await).""" + config = load_config(scenario) + print("=== Asynchronous Initialization Mode ===") + print("Connection info:") + show_active_config(config) + print() + + await AsyncUser.configure(config, AsyncPostgresBackend) + AsyncPost.__backend__ = AsyncUser.backend() + AsyncComment.__backend__ = AsyncUser.backend() + + print("Dropping old tables and creating new ones...") + backend = AsyncUser.backend() + await backend.execute(SCHEMA_SQL) + print("Tables created, inserting test data...") + + users = [] + for i in range(1, NUM_USERS + 1): + u = AsyncUser( + username=f"user{i:02d}", + email=f"user{i:02d}@example.com", + is_active=True, + ) + await u.save() + users.append(u) + print(f" User #{u.id}: {u.username}") + + post_count = 0 + comment_count = 0 + for u in users: + for j in range(1, NUM_POSTS_PER_USER + 1): + p = AsyncPost( + user_id=u.id, + title=f"{u.username}'s post #{j}", + body=f"This is the body of {u.username}'s post #{j}.", + status="published", + view_count=0, + ) + await p.save() + post_count += 1 + for k in range(1, NUM_COMMENTS_PER_POST + 1): + c = AsyncComment( + post_id=p.id, + user_id=users[(k - 1) % len(users)].id, + body=f"Comment #{k} on post #{p.id}", + is_approved=(k % 2 == 1), + ) + await c.save() + comment_count += 1 + + print(f"\n Inserted {post_count} posts") + print(f" Inserted {comment_count} comments") + + await AsyncUser.backend().disconnect() + print("\n✓ Asynchronous initialization complete") + print(f" Users {NUM_USERS} / Posts {post_count} / Comments {comment_count}") + + +# ───────────────────────────────────────────────────────────────────────────── +# Entry Point +# ───────────────────────────────────────────────────────────────────────────── + + +def main() -> None: + parser = argparse.ArgumentParser(description="Initialize parallel worker experiment database (PostgreSQL)") + parser.add_argument("--async", dest="use_async", action="store_true", help="Use asynchronous initialization mode") + parser.add_argument("--scenario", default=None, help="Specify YAML scenario name (e.g., postgres_16)") + args = parser.parse_args() + + if args.use_async: + asyncio.run(init_async(args.scenario)) + else: + init_sync(args.scenario) + + print("\nNext steps (run this script before each experiment to reset data):") + print(" python setup_db.py && python exp1_basic_multiprocess.py") + print(" python setup_db.py && python exp2_postgres_async_advantage.py") + print(" python setup_db.py && python exp3_deadlock_wrong.py") + print(" python setup_db.py && python exp4_partition_correct.py") + print(" python setup_db.py && python exp5_multithread_warning.py") + + +if __name__ == "__main__": + main() diff --git a/docs/zh_CN/README.md b/docs/zh_CN/README.md index 569750b2..b2a2e767 100644 --- a/docs/zh_CN/README.md +++ b/docs/zh_CN/README.md @@ -46,4 +46,7 @@ * **[常见连接错误](troubleshooting/connection.md)**: 连接问题诊断 * **[性能问题](troubleshooting/performance.md)**: 性能瓶颈分析 +8. **[场景实战 (Scenarios)](scenarios/README.md)** + * **[并行 Worker 处理](scenarios/parallel_workers.md)**: 多进程/异步并发场景的正确用法 + > 📖 **核心库文档**：要了解 ActiveRecord 框架的完整功能，请参考 [rhosocial-activerecord 文档](https://github.com/Rhosocial/python-activerecord/tree/main/docs/zh_CN)。 diff --git a/docs/zh_CN/scenarios/README.md b/docs/zh_CN/scenarios/README.md new file mode 100644 index 00000000..65d06129 --- /dev/null +++ b/docs/zh_CN/scenarios/README.md @@ -0,0 +1,27 @@ +# 场景实战 + +本章节提供 PostgreSQL 后端在真实业务场景中的完整应用示例，帮助开发者理解如何在实际项目中正确使用 rhosocial-activerecord。 + +## 场景列表 + +### [并行 Worker 处理](parallel_workers.md) + +演示 PostgreSQL 后端在多进程/异步并发场景下的正确用法： + +- **多进程正确用法**：`configure()` 必须在子进程内调用 +- **PostgreSQL 异步优势**：psycopg 原生异步支持，真正的网络 I/O 并发 +- **死锁处理**：PostgreSQL 自动检测死锁，生产环境推荐的重试机制 +- **多线程陷阱**：psycopg 连接同样不支持多线程并发 + +> 📖 **配套代码**：完整可运行的实验代码位于 `docs/examples/chapter_08_scenarios/parallel_workers/` 目录。 + +## 与核心库场景的关系 + +本章节是 [核心库场景文档](https://github.com/Rhosocial/python-activerecord/tree/main/docs/zh_CN/scenarios) 的 PostgreSQL 特定补充，重点关注： + +- PostgreSQL 特有的并发行为（MVCC、行级锁、死锁检测） +- psycopg 异步驱动的特点（单连接模型、真异步 I/O） +- RETURNING 子句的优势（INSERT/UPDATE/DELETE 返回数据） +- 与 MySQL 和 SQLite 的关键差异对比 + +核心库场景中介绍的通用 ActiveRecord 用法（如关联关系、查询构建）同样适用于 PostgreSQL 后端。 diff --git a/docs/zh_CN/scenarios/parallel_workers.md b/docs/zh_CN/scenarios/parallel_workers.md new file mode 100644 index 00000000..ec7d9311 --- /dev/null +++ b/docs/zh_CN/scenarios/parallel_workers.md @@ -0,0 +1,653 @@ +# 并行 Worker 场景下的正确用法（PostgreSQL） + +在数据处理、任务队列、批量导入等场景中，开发者常常希望用多个 Worker 并行处理任务以提升吞吐量。本章专注于 PostgreSQL 数据库的并行 Worker 使用模式，阐述 PostgreSQL 与 MySQL、SQLite 在并发处理上的差异，并给出经过验证的安全方案。 + +> **贯穿本章的设计原则**：`rhosocial-activerecord` 的同步类 `BaseActiveRecord` 与异步类 `AsyncBaseActiveRecord` 方法名**完全相同**——`configure()` / `backend()` / `transaction()` / `save()` 等均如此，异步版本只需加 `await` 或 `async with`。本章所有示例均提供两种版本。 + +## 目录 + +1. [PostgreSQL 并发能力概述](#1-postgresql-并发能力概述) +2. [多进程：推荐方案](#2-多进程推荐方案) +3. [PostgreSQL 异步后端的特点](#3-postgresql-异步后端的特点) +4. [死锁：PostgreSQL 自动检测与预防](#4-死锁postgresql-自动检测与预防) +5. [应用分离原则](#5-应用分离原则) +6. [asyncio 并发在 PostgreSQL 上的行为](#6-asyncio-并发在-postgresql-上的行为) +7. [示例代码](#7-示例代码) + +--- + +## 1. PostgreSQL 并发能力概述 + +### 1.1 与 MySQL、SQLite 的差异对比 + +`rhosocial-activerecord` 同样遵循**一个 ActiveRecord 类，绑定一条连接**的核心设计原则： + +- **同步**：`Post.configure(config, PostgreSQLBackend)` → 写入 `Post.__backend__` +- **异步**：`await Post.configure(config, AsyncPostgreSQLBackend)` → 写入 `Post.__backend__` + +配置写入的是**类级别属性**，多线程间仍然不安全。PostgreSQL 在以下方面与其他数据库有差异： + +| 特性 | SQLite | MySQL | PostgreSQL | +| --- | --- | --- | --- | +| 锁粒度 | 文件级锁 | 行级锁（InnoDB） | 行级锁（MVCC） | +| 并发写入 | 需配置 WAL 模式 | 默认支持 | 默认支持（MVCC） | +| 异步驱动 | `aiosqlite`：线程池模拟 | `mysql-connector-python` 原生 async | `psycopg` 原生 async | +| 死锁处理 | 超时等待 | 自动检测，errno 1213 | 自动检测，SQLSTATE 40P01 | +| 连接类型 | 文件路径（本地） | TCP 网络连接 | TCP 网络连接 | +| RETURNING | ❌ 不支持 | ❌ 不支持 | ✅ 原生支持 | +| Free-threaded | ✅ 支持 | ✅ 支持 | ❌ 不支持（psycopg 限制） | + +### 1.2 单连接模型的不变性 + +无论 PostgreSQL 有多少并发优势，**单个 `ActiveRecord` 类绑定的 `__backend__` 仍是一条连接**。在多线程环境下，并发操作同一 `__backend__` 会导致游标状态混乱。`psycopg` 官方明确说明连接不支持多线程并发使用。 + +> **不要将 ActiveRecord 配置在多个线程之间共享。** 多进程是并行 Worker 场景的正确选择。 + +### 1.3 psycopg 的 Free-threaded 限制 + +`psycopg` 版本 3 不支持 Python 3.13t/3.14t 的 free-threaded 构建。如果您的项目需要 free-threaded Python，请考虑使用 MySQL 或 SQLite 后端。 + +--- + +## 2. 多进程：推荐方案 + +多进程是并行 Worker 场景的推荐方案。每个进程拥有独立的内存空间，`configure()` 在进程内独立执行，建立独立的 TCP 连接。 + +### 2.1 正确的生命周期 + +**同步（multiprocessing）**： + +```python +import multiprocessing +from rhosocial.activerecord.backend.impl.postgres import PostgreSQLBackend, PostgreSQLConnectionConfig +from models import Comment, Post, User + +def worker(post_ids: list[int]): + # 1. 进程启动后，在进程内配置连接——每个进程建立独立 TCP 连接 + config = PostgreSQLConnectionConfig( + host="localhost", + port=5432, + database="mydb", + username="app", + password="secret", + ) + User.configure(config, PostgreSQLBackend) + Post.__backend__ = User.backend() + Comment.__backend__ = User.backend() + + try: + for post_id in post_ids: + post = Post.find_one(post_id) + if post is None: + continue + author = post.author() # BelongsTo 关联 + approved = len([c for c in post.comments() if c.is_approved]) + post.view_count = 1 + approved + post.save() + finally: + # 2. 进程退出前断开连接 + User.backend().disconnect() + + +if __name__ == "__main__": + post_ids = list(range(1, 101)) + chunk_size = 25 + + with multiprocessing.Pool(processes=4) as pool: + chunks = [post_ids[i:i+chunk_size] for i in range(0, len(post_ids), chunk_size)] + pool.map(worker, chunks) +``` + +**异步（asyncio + 多进程）**： + +```python +import asyncio +import multiprocessing +from rhosocial.activerecord.backend.impl.postgres import AsyncPostgreSQLBackend, PostgreSQLConnectionConfig +from models import AsyncComment, AsyncPost, AsyncUser + +async def async_worker_main(post_ids: list[int]): + # 1. 在进程内配置异步连接（psycopg 原生 async，TCP 网络 I/O） + config = PostgreSQLConnectionConfig( + host="localhost", port=5432, + database="mydb", username="app", password="secret", + ) + await AsyncUser.configure(config, AsyncPostgreSQLBackend) + AsyncPost.__backend__ = AsyncUser.backend() + AsyncComment.__backend__ = AsyncUser.backend() + + try: + async def process_post(post_id: int): + post = await AsyncPost.find_one(post_id) + if post is None: + return + author = await post.author() # 加 await 加括号 + approved = len([c for c in await post.comments() if c.is_approved]) + post.view_count = 1 + approved + await post.save() + + # 单连接顺序执行（进程间才是并发） + for pid in post_ids: + await process_post(pid) + finally: + await AsyncUser.backend().disconnect() + + +def run_async_worker(post_ids: list[int]): + # 每个进程有独立的 event loop 和独立的 PostgreSQL 连接 + asyncio.run(async_worker_main(post_ids)) + + +if __name__ == "__main__": + post_ids = list(range(1, 101)) + chunk_size = 25 + + with multiprocessing.Pool(processes=4) as pool: + chunks = [post_ids[i:i+chunk_size] for i in range(0, len(post_ids), chunk_size)] + pool.map(run_async_worker, chunks) +``` + +**关键规则**： + +- `configure()` 必须在子进程内调用，不能在父进程配置后 `fork` +- PostgreSQL 连接是 TCP 连接，`fork` 后继承文件描述符比 SQLite 更危险 +- 异步场景下，同一进程内的协程天然串行访问数据库（event loop 单线程调度） + +可运行的耗时对比演示见 [exp1_basic_multiprocess.py](../../examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py)。 + +--- + +## 3. PostgreSQL 异步后端的特点 + +### 3.1 单连接模型的约束 + +`rhosocial-activerecord` 的异步 PostgreSQL 后端（`AsyncPostgreSQLBackend`）基于 `psycopg` 版本 3 的异步接口，**每个 ActiveRecord 类绑定一条连接**。这与连接池方案不同： + +| 特性 | 单连接 ORM（本项目） | 连接池方案 | +| --- | --- | --- | +| 同进程内 asyncio.gather | ❌ 不支持，会引发运行时错误 | ✅ 支持，每个协程使用不同连接 | +| 配置复杂度 | 低，`configure()` 一行 | 高，需手动管理连接池 | +| 多进程并发 | ✅ 每进程独立连接，真正并发 | ✅ 同样支持 | +| 适用场景 | 批处理、任务队列、数据管道 | 高并发 Web 服务 | + +### 3.2 psycopg 异步的优势 + +`psycopg` 的异步实现基于 Python 原生 `asyncio`，在等待数据库响应时真正释放控制权： + +```text +单连接异步执行路径： + 协程 A → 发送 SQL → await → PostgreSQL 响应 → 协程 A 恢复 + 协程 B → 必须等待 A 完成后才能发送（顺序执行） + +多进程并发执行路径： + 进程 A → 独立连接 → 发送 SQL → await（并发） + 进程 B → 独立连接 → 发送 SQL → await（并发） + （两者真正同时在传输层并发） +``` + +### 3.3 多进程是并发的正确姿势 + +虽然单进程内协程必须顺序执行，但**多进程间**每个进程独立持有连接，真正并发： + +```python +async def process_posts_sequential(post_ids: list[int]): + """ + 单连接：同一进程内顺序执行协程。 + 并发通过多进程实现——每个进程独立连接。 + """ + async def update_one(post_id: int): + post = await AsyncPost.find_one(post_id) + if post: + post.view_count += 1 + await post.save() + + # 单连接：顺序执行 + for pid in post_ids: + await update_one(pid) +``` + +可运行的多进程 async vs sync 性能对比见 [exp2_postgres_async_advantage.py](../../examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py)。 + +--- + +## 4. 死锁：PostgreSQL 自动检测与预防 + +PostgreSQL 内置死锁检测算法，死锁发生时自动回滚其中一个事务，另一方继续执行。被回滚方收到 `OperationalError`（SQLSTATE 40P01）。 + +### 4.1 死锁的根本原因：行锁顺序不一致 + +```python +# ❌ 错误：不同 Worker 以相反顺序锁定行 +def worker_a(): + with Post.transaction(): + post1 = Post.find_one(1) # Worker A 先锁定 id=1 + time.sleep(0.01) + post2 = Post.find_one(2) # Worker A 再请求 id=2（此时 B 已持有） + # → PostgreSQL 检测到死锁，选择回滚 A 或 B + +def worker_b(): + with Post.transaction(): + post2 = Post.find_one(2) # Worker B 先锁定 id=2 + time.sleep(0.01) + post1 = Post.find_one(1) # Worker B 再请求 id=1（此时 A 已持有） +``` + +反面教材见 [exp3_deadlock_wrong.py](../../examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py)。 + +### 4.2 预防方案一：固定锁顺序 + +```python +# ✅ 正确：始终按主键升序锁定资源 +def transfer_safe(from_id: int, to_id: int, amount: float): + first_id, second_id = min(from_id, to_id), max(from_id, to_id) + with Account.transaction(): + first = Account.find_one(first_id) + second = Account.find_one(second_id) + debit, credit = (first, second) if from_id < to_id else (second, first) + debit.balance -= amount + credit.balance += amount + debit.save() + credit.save() + +# 异步版本（方法名相同，加 await） +async def transfer_safe_async(from_id: int, to_id: int, amount: float): + first_id, second_id = min(from_id, to_id), max(from_id, to_id) + async with Account.transaction(): + first = await Account.find_one(first_id) + second = await Account.find_one(second_id) + debit, credit = (first, second) if from_id < to_id else (second, first) + debit.balance -= amount + credit.balance += amount + await debit.save() + await credit.save() +``` + +### 4.3 预防方案二：原子领取（事务内查询 + 更新） + +```python +# ✅ 正确：在事务内原子领取，PostgreSQL 行级锁保证不重复 +def claim_posts(batch_size: int = 5) -> list: + with Post.transaction(): + pending = ( + Post.query() + .where(Post.c.status == "draft") + .order_by(Post.c.id) + .limit(batch_size) + .for_update() # PostgreSQL 支持 FOR UPDATE + .all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + post.save() + return pending + +# 异步版本（方法名相同，加 await） +async def claim_posts_async(batch_size: int = 5) -> list: + async with AsyncPost.transaction(): + pending = await ( + AsyncPost.query() + .where(AsyncPost.c.status == "draft") + .order_by(AsyncPost.c.id) + .limit(batch_size) + .for_update() + .all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + await post.save() + return pending +``` + +### 4.3.1 跨数据库兼容的能力检测模式 + +当您的代码需要在多个数据库后端之间复用时，应该先检测后端能力再使用 FOR UPDATE： + +```python +def claim_posts_portable(batch_size: int = 5) -> list: + """跨数据库兼容的原子领取实现""" + # 获取后端方言并检测能力 + dialect = Post.backend().dialect + supports_for_update = dialect.supports_for_update() + + with Post.transaction(): + query = ( + Post.query() + .where(Post.c.status == "draft") + .order_by(Post.c.id) + .limit(batch_size) + ) + + # 仅在支持的后端使用 FOR UPDATE + if supports_for_update: + query = query.for_update() + + pending = query.all() + + if not pending: + return [] + for post in pending: + post.status = "processing" + post.save() + return pending +``` + +**能力检测设计原则**： + +| 原则 | 说明 | +| --- | --- | +| **不替用户做选择** | 方言默认返回 `False`，只有明确支持的后端才返回 `True` | +| **双层检测防御** | ActiveQuery 层面调用 `for_update()` 时会检测；Dialect 层面生成 SQL 时再次检测 | +| **显式优于隐式** | 不支持 FOR UPDATE 时抛出 `UnsupportedFeatureError`，而非静默忽略 | +| **用户自主适配** | 用户通过 `supports_for_update()` 判断后，可选择替代方案（如数据分区） | + +> **PostgreSQL 说明**：PostgreSQL 从早期版本就支持 FOR UPDATE，因此 `PostgreSQLDialect.supports_for_update()` 始终返回 `True`。SQLite 不支持 FOR UPDATE（使用文件级锁），返回 `False`。 + +### 4.4 PostgreSQL 专有方案：捕获死锁并重试（生产推荐） + +PostgreSQL 有自动死锁检测，因此可以**不预防死锁，而是捕获并重试**。这是 PostgreSQL 生产环境的推荐模式： + +```python +import time + +def _is_deadlock(exc: Exception) -> bool: + """判断异常是否为 PostgreSQL 死锁（SQLSTATE 40P01）""" + # psycopg 3 的错误对象有 sqlstate 属性 + if hasattr(exc, 'sqlstate'): + return exc.sqlstate == '40P01' + msg = str(exc) + return '40P01' in msg or 'deadlock' in msg.lower() + + +def claim_posts_with_retry(batch_size: int = 5, max_retry: int = 3) -> list: + """原子领取 + 死锁自动重试""" + for attempt in range(max_retry): + try: + with Post.transaction(): + pending = ( + Post.query() + .where(Post.c.status == "draft") + .order_by(Post.c.id) + .limit(batch_size) + .for_update() + .all() + ) + if not pending: + return [] + for post in pending: + post.status = "processing" + post.save() + return pending + except Exception as e: + if _is_deadlock(e) and attempt < max_retry - 1: + time.sleep(0.05 * (attempt + 1)) # 指数退避 + continue + raise + return [] +``` + +### 4.5 五条预防原则（含 PostgreSQL 专有说明） + +| 原则 | 说明 | +| --- | --- | +| **数据分区** | 将数据集按 ID 范围或哈希分配给各 Worker，避免多进程操作同一行 | +| **一致的锁顺序** | 涉及多资源时，按固定顺序（如主键升序）请求锁 | +| **短事务** | 事务内只做必要操作，不跨越 I/O 等待或耗时计算 | +| **原子领取** | 在事务内查询并更新任务状态，使用 `FOR UPDATE` 锁定 | +| **死锁重试**（PostgreSQL 专有） | 捕获 `OperationalError`（SQLSTATE 40P01）并重试 | + +> **PostgreSQL vs MySQL 差异**：MySQL 死锁错误码为 errno 1213，PostgreSQL 为 SQLSTATE 40P01。两者都有自动死锁检测。 + +可运行的正确方案演示见 [exp4_partition_correct.py](../../examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py)。 + +--- + +## 5. 应用分离原则 + +当系统同时包含两种截然不同的工作负载时，建议将它们部署为独立应用。 + +### 5.1 场景示例 + +| 工作负载类型 | 特征 | 合适部署 | +| --- | --- | --- | +| Web API 服务 | 短请求、高并发、响应时间敏感 | FastAPI / Django + asyncio + 连接池 | +| 数据分析批处理 | 长时间运行、大数据量、CPU 密集 | 独立脚本 + multiprocessing + PostgreSQLBackend | +| 任务队列消费 | 定期轮询、任务间独立、易于水平扩展 | 独立 Worker 进程池 | + +### 5.2 推荐架构 + +```text +用户请求 ──→ Web 应用（asyncio + 连接池） + │ + └──→ 任务队列（PostgreSQL 数据库表 / Redis） + │ + └──→ 后台 Worker 进程池 + （每进程独立 PostgreSQLBackend 同步连接） +``` + +Web 应用负责接收请求、写入任务队列；Worker 进程池负责执行耗时任务。 + +--- + +## 6. asyncio 并发在 PostgreSQL 上的行为 + +### 6.1 单连接模型的限制 + +`rhosocial-activerecord` 遵循**一个 ActiveRecord 类绑定一条连接**的核心设计原则。这意味着同一进程内的协程**不能**通过 `asyncio.gather` 并发访问数据库——单个连接在同一时刻只能处理一个查询： + +```python +# ❌ 错误：单连接不支持并发访问 +async def batch_update_wrong(post_ids: list[int]): + await asyncio.gather(*[ + update_one(pid) for pid in post_ids # 会引发运行时错误 + ]) + +# ✅ 正确：单连接内顺序执行 +async def batch_update_correct(post_ids: list[int]): + for pid in post_ids: + await update_one(pid) +``` + +### 6.2 多进程是并发的载体 + +异步的优势在于**多进程间**：每个进程持有独立的 PostgreSQL TCP 连接，进程间真正并发。 + +### 6.3 PostgreSQL 异步与 MySQL 异步的对比 + +| 场景 | PostgreSQL（psycopg 3） | MySQL（mysql-connector-python） | +| --- | --- | --- | +| 同进程内 asyncio.gather | ❌ 单连接不支持 | ❌ 单连接不支持 | +| 多进程并发写入 | ✅ MVCC，无需配置 | ✅ 行级锁，无需配置 | +| 死锁处理 | ✅ 自动检测，SQLSTATE 40P01 | ✅ 自动检测，errno 1213 | +| Free-threaded 支持 | ❌ 不支持 | ✅ 支持 | + +--- + +## 7. 示例代码 + +本章的完整可运行示例位于 [`docs/examples/chapter_08_scenarios/parallel_workers/`](../../examples/chapter_08_scenarios/parallel_workers/)，包含以下实验： + +| 文件 | 内容 | 对应章节 | +| --- | --- | --- | +| [`config_loader.py`](../../examples/chapter_08_scenarios/parallel_workers/config_loader.py) | 连接配置加载（YAML / 环境变量 / 默认值） | — | +| [`models.py`](../../examples/chapter_08_scenarios/parallel_workers/models.py) | 共享模型定义（`User`、`Post`、`Comment` 同步版 + 异步版） | — | +| [`setup_db.py`](../../examples/chapter_08_scenarios/parallel_workers/setup_db.py) | 数据库初始化脚本（同步/异步两种模式） | — | +| [`exp1_basic_multiprocess.py`](../../examples/chapter_08_scenarios/parallel_workers/exp1_basic_multiprocess.py) | 正确的多进程用法（含串行/同步多进程/异步多进程耗时对比） | §2.1 | +| [`exp2_postgres_async_advantage.py`](../../examples/chapter_08_scenarios/parallel_workers/exp2_postgres_async_advantage.py) | PostgreSQL 异步特点：单连接限制说明 + 多进程并发写入对比 | §3 | +| [`exp3_deadlock_wrong.py`](../../examples/chapter_08_scenarios/parallel_workers/exp3_deadlock_wrong.py) | 行锁顺序冲突导致 PostgreSQL 死锁（反面教材） | §4.1 | +| [`exp4_partition_correct.py`](../../examples/chapter_08_scenarios/parallel_workers/exp4_partition_correct.py) | 数据分区 + 原子领取 + 死锁重试（同步/异步各方案） | §4.2–4.4 | +| [`exp5_multithread_warning.py`](../../examples/chapter_08_scenarios/parallel_workers/exp5_multithread_warning.py) | 多线程共享连接的问题（反面教材） | §1.2 | + +> **说明**：所有示例文件均直接使用 `rhosocial-activerecord` ORM，模型体系为 `User → Post → Comment`，并体现 `HasMany` / `BelongsTo` 关联关系的同步与异步对等用法。 + +运行前请先执行初始化脚本： + +```bash +cd docs/examples/chapter_08_scenarios/parallel_workers +python setup_db.py +python exp1_basic_multiprocess.py # 运行任意实验 +``` + +详见该目录下的 `README.md` 了解各实验的完整说明和预期输出。 + +--- + +## 8. WorkerPool 测试经验总结 + +### 8.1 异步 Worker 测试的已知限制 + +在 `WorkerPool` 多进程环境下运行异步测试时，存在以下已知限制： + +#### Event Loop 跨进程问题 + +当 `WorkerPool` 在子进程中执行异步任务时，每个子进程通过 `asyncio.run()` 创建独立的 event loop。然而，测试框架（pytest-asyncio）的 fixture 在主进程中创建的 event loop 与子进程中的 event loop 是隔离的： + +```text +主进程（pytest）： + └── Event Loop A（pytest-asyncio 创建） + └── Fixture: async_user_class_for_worker + └── 异步后端实例绑定到 Loop A + +子进程（Worker）： + └── Event Loop B（asyncio.run() 创建） + └── 任务尝试使用绑定到 Loop A 的异步后端 + └── 错误：Task got Future attached to a different loop +``` + +**错误示例**： + +```python +# ❌ 错误：在主进程 fixture 中创建的异步后端无法在子进程中使用 +async def async_worker_task(user_id, conn_params): + # conn_params 包含的异步后端实例绑定到主进程的 event loop + # 子进程尝试使用时会失败 + backend = conn_params['backend'] # 绑定到错误的 loop + user = await backend.find_one(user_id) # RuntimeError! +``` + +**正确做法**： + +```python +# ✅ 正确：在子进程内部创建新的异步后端实例 +async def async_worker_task(user_id, conn_params): + # 只传递连接参数（可序列化），在子进程内创建新实例 + config = conn_params['config_kwargs'] + await Model.configure(config, AsyncPostgreSQLBackend) + user = await Model.find_one(user_id) + await Model.backend().disconnect() +``` + +#### 受影响的测试场景 + +| 测试类型 | 同步版本 | 异步版本 | 原因 | +|----------|----------|----------|------| +| 并行读取 | ✅ 通过 | ❌ 失败 | 异步后端绑定到主进程 loop | +| 并行更新 | ✅ 通过 | ❌ 失败 | 同上 | +| 并行删除 | ✅ 通过 | ❌ 失败 | 同上 | +| 并行查询 | ✅ 通过 | ❌ 失败 | 同上 | +| 事务隔离 | ✅ 通过 | ❌ 失败 | 同上 + 事务状态跨进程问题 | +| Worker 生命周期 | ✅ 通过 | ✅ 通过 | 不涉及跨进程异步操作 | +| 连接管理 | ✅ 通过 | ✅ 通过 | 不涉及跨进程异步操作 | + +### 8.2 测试覆盖率说明 + +Worker 测试对 PostgreSQL 后端的覆盖率贡献： + +| 模块 | 覆盖率 | 说明 | +|----------|--------|------| +| `backend.py` | ~32% | 同步后端核心功能 | +| `async_backend.py` | ~35% | 异步后端核心功能 | +| `mixins/` | ~18% | DML 操作 | +| `dialect.py` | ~22% | SQL 方言处理 | + +同步测试覆盖率较高，异步测试因 event loop 问题导致部分代码路径未被覆盖。 + +### 8.3 生产环境建议 + +1. **同步 Worker 是首选**：在多进程 Worker 场景下，同步后端（`PostgreSQLBackend`）是更稳定的选择 +2. **异步适用于单进程**：异步后端（`AsyncPostgreSQLBackend`）在单进程内的顺序执行场景下表现良好 +3. **避免跨进程传递异步实例**：只传递可序列化的连接参数，在子进程内创建新的异步后端实例 + +### 8.4 FOR UPDATE 能力检测经验 + +#### 问题背景 + +在 WorkerPool 多进程测试中，`test_transaction_isolation.py` 的转账测试使用了 `FOR UPDATE` 子句来锁定行。然而，SQLite 后端不支持 `FOR UPDATE`（使用文件级锁），导致测试失败。 + +#### 解决方案：能力检测模式 + +遵循「不替用户做选择」的设计原则，我们实现了两层能力检测： + +**1. Dialect 层面的能力声明**： + +```python +# SQLDialectBase（默认不支持） +def supports_for_update(self) -> bool: + """默认返回 False，只有支持的后端才重写此方法""" + return False + +# PostgreSQLDialect（明确支持） +def supports_for_update(self) -> bool: + return True +``` + +**2. ActiveQuery 层面的早期检测**： + +```python +# ActiveQuery.for_update() 方法中 +def for_update(self, ...): + if not dialect.supports_for_update(): + raise UnsupportedFeatureError( + dialect.name, + "FOR UPDATE clause", + "This backend does not support row-level locking with FOR UPDATE. " + "Use dialect.supports_for_update() to check support before calling this method." + ) +``` + +**3. Dialect 层面的安全网**： + +```python +# SQLDialectBase.format_query_statement() 中 +if expr.for_update: + if not self.supports_for_update(): + raise UnsupportedFeatureError(...) + # 生成 FOR UPDATE SQL... +``` + +#### 测试代码适配 + +测试任务函数中使用能力检测来适配不同后端： + +```python +def transfer_task(from_id: int, to_id: int, amount: float, conn_params: dict): + # 检测后端是否支持 FOR UPDATE + supports_for_update = backend.dialect.supports_for_update() + + with Model.transaction(): + if supports_for_update: + # PostgreSQL/MySQL：使用 FOR UPDATE 锁定 + first = Model.query().where(Model.c.id == first_id).for_update().one() + else: + # SQLite：使用普通查询（依赖文件锁） + first = Model.find_one({'id': first_id}) + # ... 业务逻辑 +``` + +#### 设计经验总结 + +| 经验 | 说明 | +|------|------| +| **默认拒绝原则** | 基类返回 `False`，后端必须显式声明支持 | +| **不替用户做选择** | 不支持时抛错，而非静默忽略（Django/Rails 方案） | +| **双层防御** | ActiveQuery 提供早期失败，Dialect 作为安全网 | +| **用户自主适配** | 用户通过 `supports_for_update()` 判断后选择替代方案 | + +### 8.5 相关文件 + +- 测试桥接文件：`tests/rhosocial/activerecord_postgres_test/feature/basic/worker/` +- Provider 实现：`tests/providers/basic.py`、`tests/providers/query.py` +- WorkerPool 实现：`rhosocial.activerecord.worker.pool` diff --git a/src/rhosocial/activerecord/backend/impl/postgres/dialect.py b/src/rhosocial/activerecord/backend/impl/postgres/dialect.py index 697b3b94..1b647ca0 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/dialect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/dialect.py @@ -436,6 +436,16 @@ def supports_graph_match(self) -> bool: # Though graph querying can be done with extensions like Apache AGE return False + def supports_for_update(self) -> bool: + """Whether FOR UPDATE clause is supported in SELECT statements. + + PostgreSQL supports FOR UPDATE since early versions. The clause locks + selected rows preventing other transactions from modifying them. + PostgreSQL also supports FOR UPDATE OF, FOR UPDATE NOWAIT, and + FOR UPDATE SKIP LOCKED (since 9.5). + """ + return True + def supports_for_update_skip_locked(self) -> bool: """Whether FOR UPDATE SKIP LOCKED is supported.""" return self.version >= (9, 5, 0) # Supported since 9.5 diff --git a/tests/providers/basic.py b/tests/providers/basic.py index 0c6cc8cb..ee1eb06b 100644 --- a/tests/providers/basic.py +++ b/tests/providers/basic.py @@ -20,6 +20,7 @@ from rhosocial.activerecord.model import ActiveRecord from rhosocial.activerecord.backend.type_adapter import BaseSQLTypeAdapter from rhosocial.activerecord.testsuite.feature.basic.interfaces import IBasicProvider +from rhosocial.activerecord.testsuite.core.protocols import WorkerTestProtocol # Import the fixture selector utility from rhosocial.activerecord.testsuite.utils import select_fixture @@ -166,7 +167,7 @@ def _select_model_class(base_cls, py312_cls, py311_cls, py310_cls, model_name: s from .scenarios import get_enabled_scenarios, get_scenario -class BasicProvider(IBasicProvider): +class BasicProvider(IBasicProvider, WorkerTestProtocol): """ This is the postgres backend's implementation for the basic features test group. It connects the generic tests in the testsuite with the actual postgres database. @@ -430,3 +431,72 @@ def cleanup_after_test(self, scenario_name: str): # Clear the list of active backends for the next test self._active_backends.clear() + + # --- Implementation of WorkerTestProtocol --- + + def get_worker_connection_params(self, scenario_name: str, fixture_type: str = None) -> dict: + """ + Return serializable connection parameters for Worker processes. + + This method provides all information needed to recreate the database + connection in a Worker process, including the schema SQL for table creation. + + Args: + scenario_name: The test scenario name + fixture_type: Optional fixture type hint (e.g., 'user', 'async_user') + Used to determine if async backend is needed + + Returns: + Dictionary with connection parameters and schema SQL + """ + from .scenarios import SCENARIO_MAP + + # Determine if async backend is needed based on fixture_type + is_async = fixture_type and fixture_type.startswith('async_') + backend_class_name = 'AsyncPostgresBackend' if is_async else 'PostgresBackend' + + # Determine schema file based on fixture_type + table_name = 'users' # default + if fixture_type: + # Remove 'async_' prefix if present to get the base type + base_type = fixture_type.replace('async_', '') + table_map = { + 'user': 'users', + 'type_case': 'type_cases', + 'type_test': 'type_tests', + 'validated_field_user': 'validated_field_users', + 'validated_user': 'validated_users', + 'type_adapter_test': 'type_adapter_tests', + } + table_name = table_map.get(base_type, 'users') + + # Get connection config from scenario + if scenario_name not in SCENARIO_MAP: + if SCENARIO_MAP: + scenario_name = next(iter(SCENARIO_MAP)) + else: + raise ValueError("No scenarios registered") + + config_dict = SCENARIO_MAP[scenario_name] + + return { + 'backend_module': 'rhosocial.activerecord.backend.impl.postgres', + 'backend_class_name': backend_class_name, + 'config_class_module': 'rhosocial.activerecord.backend.impl.postgres.config', + 'config_class_name': 'PostgresConnectionConfig', + 'config_kwargs': config_dict, + 'schema_sql': self._load_postgres_schema(f'{table_name}.sql'), + } + + def get_worker_schema_sql(self, scenario_name: str, table_name: str) -> str: + """ + Return the SQL statement to create a specific table. + + Args: + scenario_name: The test scenario name (unused for Postgres as schema is fixed) + table_name: Name of the table to create + + Returns: + CREATE TABLE SQL statement + """ + return self._load_postgres_schema(f'{table_name}.sql') diff --git a/tests/providers/query.py b/tests/providers/query.py index 00af00f1..77687471 100644 --- a/tests/providers/query.py +++ b/tests/providers/query.py @@ -100,11 +100,12 @@ def _select_model_class(base_cls, py312_cls, py311_cls, py310_cls, model_name: s MappedComment = _select_model_class(MappedCommentBase, MappedComment312, MappedComment311, MappedComment310, "MappedComment") from rhosocial.activerecord.testsuite.feature.query.interfaces import IQueryProvider +from rhosocial.activerecord.testsuite.core.protocols import WorkerTestProtocol # The scenarios are defined specifically for this backend. from .scenarios import get_enabled_scenarios, get_scenario -class QueryProvider(IQueryProvider): +class QueryProvider(IQueryProvider, WorkerTestProtocol): """ This is the postgres backend's implementation for the query features test group. It connects the generic tests in the testsuite with the actual postgres database. @@ -396,3 +397,91 @@ def cleanup_after_test(self, scenario_name: str): # Clear the list of active backends for the next test self._active_backends.clear() + + # --- Implementation of WorkerTestProtocol --- + + def get_worker_connection_params(self, scenario_name: str, fixture_type: str = 'order') -> dict: + """ + Return serializable connection parameters for Worker processes. + + This method provides all information needed to recreate the database + connection in a Worker process, including the schema SQL for table creation. + + Args: + scenario_name: The test scenario name + fixture_type: Type of fixture ('order', 'blog', 'user', 'combined', + or with 'async_' prefix for async backends) + + Returns: + Dictionary with connection parameters and schema SQL + """ + from .scenarios import SCENARIO_MAP + + # Determine if async backend is needed based on fixture_type + is_async = fixture_type and fixture_type.startswith('async_') + backend_class_name = 'AsyncPostgresBackend' if is_async else 'PostgresBackend' + + # Get base fixture type (remove 'async_' prefix if present) + base_fixture_type = fixture_type.replace('async_', '') if fixture_type else 'order' + + # Build schema SQL based on fixture type + schema_sql = self._get_schema_sql_for_fixture_type(base_fixture_type) + + # Get connection config from scenario + if scenario_name not in SCENARIO_MAP: + if SCENARIO_MAP: + scenario_name = next(iter(SCENARIO_MAP)) + else: + raise ValueError("No scenarios registered") + + config_dict = SCENARIO_MAP[scenario_name] + + return { + 'backend_module': 'rhosocial.activerecord.backend.impl.postgres', + 'backend_class_name': backend_class_name, + 'config_class_module': 'rhosocial.activerecord.backend.impl.postgres.config', + 'config_class_name': 'PostgresConnectionConfig', + 'config_kwargs': config_dict, + 'schema_sql': schema_sql, + } + + def get_worker_schema_sql(self, scenario_name: str, table_name: str) -> str: + """ + Return the SQL statement to create a specific table. + + Args: + scenario_name: The test scenario name (unused for Postgres as schema is fixed) + table_name: Name of the table to create + + Returns: + CREATE TABLE SQL statement + """ + return self._load_postgres_schema(f'{table_name}.sql') + + def _get_schema_sql_for_fixture_type(self, fixture_type: str) -> dict: + """ + Get schema SQL for a specific fixture type. + + Args: + fixture_type: Type of fixture ('order', 'blog', 'user', 'combined') + + Returns: + Dictionary mapping table names to CREATE TABLE statements + """ + schemas = {} + + if fixture_type == 'order': + tables = ['users', 'orders', 'order_items'] + elif fixture_type == 'blog': + tables = ['users', 'posts', 'comments'] + elif fixture_type == 'user': + tables = ['users'] + elif fixture_type == 'combined': + tables = ['users', 'orders', 'order_items', 'posts', 'comments'] + else: + tables = ['users'] + + for table in tables: + schemas[table] = self._load_postgres_schema(f'{table}.sql') + + return schemas diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py new file mode 100644 index 00000000..22ae1ff1 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py @@ -0,0 +1,2 @@ +# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py +"""Worker tests for PostgreSQL backend - basic feature.""" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py new file mode 100644 index 00000000..918cae3a --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py @@ -0,0 +1,9 @@ +# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py +""" +Pytest configuration for basic worker tests. + +This file re-exports fixtures from the corresponding testsuite. +""" +# Use pytest_plugins to avoid ImportPathMismatchError when multiple +# worker/conftest.py files exist in different directories +pytest_plugins = ["rhosocial.activerecord.testsuite.feature.basic.worker.conftest"] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py new file mode 100644 index 00000000..24aa3efa --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py @@ -0,0 +1,10 @@ +# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py +""" +Bridge file for connection management tests. + +Imports tests from testsuite and makes them discoverable by pytest. +""" +from rhosocial.activerecord.testsuite.feature.basic.worker.conftest import ( + worker_connection_params, +) +from rhosocial.activerecord.testsuite.feature.basic.worker.test_connection_management import * diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_parallel_crud.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_parallel_crud.py new file mode 100644 index 00000000..734b11b3 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_parallel_crud.py @@ -0,0 +1,11 @@ +# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_parallel_crud.py +""" +Bridge file for parallel CRUD worker tests. + +Imports tests from testsuite and makes them discoverable by pytest. +""" +from rhosocial.activerecord.testsuite.feature.basic.worker.conftest import ( + user_class_for_worker, + async_user_class_for_worker, +) +from rhosocial.activerecord.testsuite.feature.basic.worker.test_parallel_crud import * diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py new file mode 100644 index 00000000..ad2ed7d4 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py @@ -0,0 +1,10 @@ +# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py +""" +Bridge file for worker pool lifecycle tests. + +Imports tests from testsuite and makes them discoverable by pytest. +""" +from rhosocial.activerecord.testsuite.feature.basic.worker.conftest import ( + worker_connection_params, +) +from rhosocial.activerecord.testsuite.feature.basic.worker.test_worker_pool_lifecycle import * diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py new file mode 100644 index 00000000..6cdfd1e9 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py @@ -0,0 +1,2 @@ +# tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py +"""Worker tests for PostgreSQL backend - query feature.""" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py new file mode 100644 index 00000000..f07fc094 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py @@ -0,0 +1,9 @@ +# tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py +""" +Pytest configuration for query worker tests. + +This file re-exports fixtures from the corresponding testsuite. +""" +# Use pytest_plugins to avoid ImportPathMismatchError when multiple +# worker/conftest.py files exist in different directories +pytest_plugins = ["rhosocial.activerecord.testsuite.feature.query.worker.conftest"] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_parallel_queries.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_parallel_queries.py new file mode 100644 index 00000000..b5c81dbb --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_parallel_queries.py @@ -0,0 +1,11 @@ +# tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_parallel_queries.py +""" +Bridge file for parallel queries worker tests. + +Imports tests from testsuite and makes them discoverable by pytest. +""" +from rhosocial.activerecord.testsuite.feature.query.worker.conftest import ( + order_fixtures_for_worker, + async_order_fixtures_for_worker, +) +from rhosocial.activerecord.testsuite.feature.query.worker.test_parallel_queries import * diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py new file mode 100644 index 00000000..c0943d2c --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py @@ -0,0 +1,11 @@ +# tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py +""" +Bridge file for transaction isolation worker tests. + +Imports tests from testsuite and makes them discoverable by pytest. +""" +from rhosocial.activerecord.testsuite.feature.query.worker.conftest import ( + order_fixtures_for_worker, + async_order_fixtures_for_worker, +) +from rhosocial.activerecord.testsuite.feature.query.worker.test_transaction_isolation import * From 757f03ed3b505d1bd908608b27d66b47091b3474 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 5 Apr 2026 15:20:05 +0800 Subject: [PATCH 06/11] fix: fix worker test directory issues for pytest 8.x compatibility - Remove conftest.py files with pytest_plugins (not allowed in non-top-level) - Remove __init__.py files that cause module resolution conflicts - Fix bridge files to import correct fixtures (user_class_for_worker) --- .../feature/basic/worker/__init__.py | 2 -- .../feature/basic/worker/conftest.py | 9 --------- .../feature/basic/worker/test_connection_management.py | 2 +- .../feature/basic/worker/test_worker_pool_lifecycle.py | 2 +- .../feature/query/worker/__init__.py | 2 -- .../feature/query/worker/conftest.py | 9 --------- 6 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py delete mode 100644 tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py delete mode 100644 tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py delete mode 100644 tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py deleted file mode 100644 index 22ae1ff1..00000000 --- a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/__init__.py -"""Worker tests for PostgreSQL backend - basic feature.""" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py deleted file mode 100644 index 918cae3a..00000000 --- a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -# tests/rhosocial/activerecord_postgres_test/feature/basic/worker/conftest.py -""" -Pytest configuration for basic worker tests. - -This file re-exports fixtures from the corresponding testsuite. -""" -# Use pytest_plugins to avoid ImportPathMismatchError when multiple -# worker/conftest.py files exist in different directories -pytest_plugins = ["rhosocial.activerecord.testsuite.feature.basic.worker.conftest"] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py index 24aa3efa..522a22fc 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_connection_management.py @@ -5,6 +5,6 @@ Imports tests from testsuite and makes them discoverable by pytest. """ from rhosocial.activerecord.testsuite.feature.basic.worker.conftest import ( - worker_connection_params, + user_class_for_worker, ) from rhosocial.activerecord.testsuite.feature.basic.worker.test_connection_management import * diff --git a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py index ad2ed7d4..af80c997 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/basic/worker/test_worker_pool_lifecycle.py @@ -5,6 +5,6 @@ Imports tests from testsuite and makes them discoverable by pytest. """ from rhosocial.activerecord.testsuite.feature.basic.worker.conftest import ( - worker_connection_params, + user_class_for_worker, ) from rhosocial.activerecord.testsuite.feature.basic.worker.test_worker_pool_lifecycle import * diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py deleted file mode 100644 index 6cdfd1e9..00000000 --- a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# tests/rhosocial/activerecord_postgres_test/feature/query/worker/__init__.py -"""Worker tests for PostgreSQL backend - query feature.""" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py deleted file mode 100644 index f07fc094..00000000 --- a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -# tests/rhosocial/activerecord_postgres_test/feature/query/worker/conftest.py -""" -Pytest configuration for query worker tests. - -This file re-exports fixtures from the corresponding testsuite. -""" -# Use pytest_plugins to avoid ImportPathMismatchError when multiple -# worker/conftest.py files exist in different directories -pytest_plugins = ["rhosocial.activerecord.testsuite.feature.query.worker.conftest"] From 2f5200ae57c7af62b4e37f95ce7ce1be6d7f3ad6 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 5 Apr 2026 15:24:20 +0800 Subject: [PATCH 07/11] fix: rename worker test file to avoid name conflict Renamed test_transaction_isolation.py to test_worker_transaction_isolation.py to avoid conflict with feature/backend/test_transaction_isolation.py --- ...ction_isolation.py => test_worker_transaction_isolation.py} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename tests/rhosocial/activerecord_postgres_test/feature/query/worker/{test_transaction_isolation.py => test_worker_transaction_isolation.py} (78%) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_worker_transaction_isolation.py similarity index 78% rename from tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py rename to tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_worker_transaction_isolation.py index c0943d2c..11f422aa 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_worker_transaction_isolation.py @@ -1,8 +1,9 @@ -# tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_transaction_isolation.py +# tests/rhosocial/activerecord_postgres_test/feature/query/worker/test_worker_transaction_isolation.py """ Bridge file for transaction isolation worker tests. Imports tests from testsuite and makes them discoverable by pytest. +Note: Renamed to avoid conflict with feature/backend/test_transaction_isolation.py """ from rhosocial.activerecord.testsuite.feature.query.worker.conftest import ( order_fixtures_for_worker, From 0588eed4e2699eb5bfd52e379567066a1ac95a4d Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 5 Apr 2026 23:19:06 +0800 Subject: [PATCH 08/11] ci: trigger CI after core package update From f99aa772728485d3bafcf4d3821d7d8fae2878ac Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 6 Apr 2026 01:24:12 +0800 Subject: [PATCH 09/11] ci: switch testsuite from git to PyPI (v1.0.0.dev8) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bfa249ad..f066c1cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: - name: Install activerecord and testsuite from PyPI run: | pip install git+https://github.com/rhosocial/python-activerecord@release/v1.0.0.dev21 - pip install git+https://github.com/rhosocial/python-activerecord-testsuite@release/v1.0.0.dev8 + pip install rhosocial-activerecord-testsuite==1.0.0.dev8 - name: Create postgres scenario config run: | From e215146b0934caab10f24a3e8522231d2a18a87b Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 6 Apr 2026 01:57:26 +0800 Subject: [PATCH 10/11] ci: use PyPI source for rhosocial-activerecord dependency Change installation source from GitHub to PyPI for rhosocial-activerecord, fixed at version 1.0.0.dev21. This simplifies dependency management and aligns with the package being published to PyPI. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f066c1cb..d09f635e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: - name: Install activerecord and testsuite from PyPI run: | - pip install git+https://github.com/rhosocial/python-activerecord@release/v1.0.0.dev21 + pip install rhosocial-activerecord==1.0.0.dev21 pip install rhosocial-activerecord-testsuite==1.0.0.dev8 - name: Create postgres scenario config From 13aa6a40691e15491e978f23cb6b7306e0bce8e2 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 6 Apr 2026 02:15:30 +0800 Subject: [PATCH 11/11] docs: update CHANGELOG for v1.0.0.dev8 --- CHANGELOG.md | 8 ++++++++ changelog.d/16.added.md | 1 - changelog.d/17.added.md | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/16.added.md delete mode 100644 changelog.d/17.added.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c2606b03..5129c433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [v1.0.0.dev8] - 2026-04-06 + +### Added + +- Added EXPLAIN clause support with typed PostgresExplainResult and dual-layer connection recovery for PostgreSQL backend ([#16](https://github.com/rhosocial/python-activerecord-postgres/issues/16)) +- Improved PostgreSQL backend CLI info command with connection status display and added comprehensive introspect usage examples. ([#17](https://github.com/rhosocial/python-activerecord-postgres/issues/17)) + + ## [v1.0.0.dev7] - 2026-03-28 ### Added diff --git a/changelog.d/16.added.md b/changelog.d/16.added.md deleted file mode 100644 index 8e871dad..00000000 --- a/changelog.d/16.added.md +++ /dev/null @@ -1 +0,0 @@ -Added EXPLAIN clause support with typed PostgresExplainResult and dual-layer connection recovery for PostgreSQL backend diff --git a/changelog.d/17.added.md b/changelog.d/17.added.md deleted file mode 100644 index 20b0db81..00000000 --- a/changelog.d/17.added.md +++ /dev/null @@ -1 +0,0 @@ -Improved PostgreSQL backend CLI info command with connection status display and added comprehensive introspect usage examples.