diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..2e723bf --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,98 @@ +name: Python Package Publish + +on: + # 手动触发发布工作流 + workflow_dispatch: + inputs: + destination: + description: '发布目标' + required: true + default: 'testpypi' + type: choice + options: + - testpypi + - pypi + +permissions: + contents: write + packages: write + +jobs: + # 检查测试工作流是否成功完成 + check-tests: + runs-on: ubuntu-latest + if: > + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || + github.event_name == 'workflow_dispatch' + steps: + - name: Success check + run: echo "前置测试通过,可以继续执行发布流程" + + # 发布到TestPyPI + publish-testpypi: + name: Publish to TestPyPI + needs: check-tests + if: > + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || + (github.event_name == 'workflow_dispatch' && github.event.inputs.destination == 'testpypi') + runs-on: ubuntu-latest + environment: publish-testpypi # 使用需要审批的环境 + permissions: + # Required for trusted publishing + id-token: write + + steps: + - name: Check out the repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + # Note: removed user and password fields to use Trusted Publishers + + # 发布到PyPI + publish-pypi: + name: Publish to PyPI + needs: check-tests + if: github.event_name == 'workflow_dispatch' && github.event.inputs.destination == 'pypi' + runs-on: ubuntu-latest + environment: publish-pypi # 使用需要严格审批的环境 + permissions: + # Required for trusted publishing + id-token: write + + steps: + - name: Check out the repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + # Note: removed user and password fields to use Trusted Publishers \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c83b008..0e870a0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,7 +80,7 @@ jobs: uses: actions/checkout@v6 with: repository: rhosocial/python-activerecord-testsuite - ref: feature/expression-schema-object-refactor + ref: test/typed-column-contracts path: _testsuite - name: Install activerecord and testsuite (editable) diff --git a/AGENTS.md b/AGENTS.md index dde030e..fba6954 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,9 @@ and raise `UnsupportedFeatureError` instead of being emulated: - MySQL-style spatial types (`GEOMETRY`/`POINT`/...) and the `ST_*` function family - MySQL 9.0 `VECTOR` type and `STRING_TO_VECTOR` / `VECTOR_TO_STRING` / - `VECTOR_DIM` / `DISTANCE_*` functions (use `Array(Float32)` + `L2Distance`) + `VECTOR_DIM` / `DISTANCE_*` functions — no such API here, by design; ClickHouse's + own vector type is `QBit`, rendered as `ClickHouseVectorType` → + `QBit(Float32, n)`, and `Array(Float32)` + `L2Distance` also works - MySQL `SET` type and `FIND_IN_SET` - Stored procedures / stored functions / `CALL` - `LOAD DATA INFILE` / `LOAD XML` diff --git a/CLAUDE.md b/CLAUDE.md index 3b9e61d..9240a9d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,9 @@ and raise `UnsupportedFeatureError` instead of being emulated: - MySQL-style spatial types (`GEOMETRY`/`POINT`/...) and the `ST_*` function family - MySQL 9.0 `VECTOR` type and `STRING_TO_VECTOR` / `VECTOR_TO_STRING` / - `VECTOR_DIM` / `DISTANCE_*` functions (use `Array(Float32)` + `L2Distance`) + `VECTOR_DIM` / `DISTANCE_*` functions — no such API here, by design; ClickHouse's + own vector type is `QBit`, rendered as `ClickHouseVectorType` → + `QBit(Float32, n)`, and `Array(Float32)` + `L2Distance` also works - MySQL `SET` type and `FIND_IN_SET` - Stored procedures / stored functions / `CALL` - `LOAD DATA INFILE` / `LOAD XML` diff --git a/README.md b/README.md index 1ba7acb..8c8c77f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ _ec7d664_ - **CI-tested server versions**: 25.8, 26.3, 26.7 - **Published**: _not yet on PyPI_ +The `<3.15` ceiling is inherited from the driver: `clickhouse-connect` declares +`Requires-Python: <3.15,>=3.10`, which is why `requires-python` is pinned to +`>=3.10,<3.15` in `pyproject.toml` rather than chosen here. The pin moves when +`clickhouse-connect` lifts its own bound. + ## Installation ```bash @@ -140,9 +145,35 @@ than being silently emulated. version. Beyond the ones in the table, that includes: `check_constraint`, `create_sequence`, `drop_sequence`, `auto_increment`, `clone`, `call`, `binlog`, `analyze_table`, `check_table`, `checksum_table`, `descending_index`, -`add_constraint`, `drop_constraint`, `add_partition`, `attach_partition`, -`detach_partition`, `deferrable_*`, and every MySQL spatial `ST_*` surface -plus the MySQL 9.0 `VECTOR` type. +`add_constraint`, `drop_constraint`, `deferrable_*`, and every MySQL spatial +`ST_*` surface plus the MySQL 9.0 `VECTOR` type. + +## Partitioning + +ClickHouse partitions a `MergeTree` table with `PARTITION BY ` in +`CREATE TABLE` — any expression over the table's columns, or a tuple of them — +declared through `storage_options` alongside `ENGINE` / `ORDER BY`. There is no +partitioning strategy to choose, no `PARTITION ... VALUES LESS THAN` / +`VALUES IN` / `MAXVALUE` boundary and no subpartitioning; a partition comes +into existence when a row lands in it. Partitions are introspected through +`system.parts` (`partition`, `partition_id`, `name`) and maintained by +partition id: + +```python +from rhosocial.activerecord.backend.impl.clickhouse.expression import ( + ClickHouseDetachPartitionExpression, # ... also DROP / ATTACH +) +sql, params = ClickHouseDetachPartitionExpression(dialect, "visits", "202601").to_sql() +# ALTER TABLE `visits` DETACH PARTITION ID '202601' +``` + +`DROP` / `DETACH` / `ATTACH` partition are the only partition clauses this +backend renders. ClickHouse has no `ADD PARTITION`, `TRUNCATE PARTITION`, +`REORGANIZE PARTITION`, `EXCHANGE PARTITION`, `SUBPARTITION BY` or +`ANALYZE`/`CHECK`/`OPTIMIZE`/`REBUILD`/`REPAIR PARTITION`; the full inventory +is at . +Empty a partition with `ALTER TABLE ... DELETE IN PARTITION ... WHERE ...`, and +a table with `TRUNCATE TABLE`. ## Version Gates diff --git a/changelog.d/5.added.md b/changelog.d/5.added.md new file mode 100644 index 0000000..6479439 --- /dev/null +++ b/changelog.d/5.added.md @@ -0,0 +1 @@ +Register the column-type protocol and render the new pad/repeat/trim nodes. `ClickHouseColumnTypeMixin` answers `suggested_column_types()` for all eighteen common Python types — every entry a column class, none refused — with each cell backed by a probe against ClickHouse 26.7.3.19, because core's rebuild removed the neutral table this backend used to derive from. `ClickHouseTrimMixin` (`mixins/trim.py`) is written from scratch: the ANSI `TRIM(BOTH FROM x)` form the shared renderer emits is a syntax error on this server (measured on the public playground), so the direction moves into the function name and the operand order becomes target-first, characters-second — `BOTH → trimBoth(x[, chars])`, `LEADING → trimLeft`, `TRAILING → trimRight`. `LpadMixin` / `RpadMixin` / `RepeatMixin` need no override, because ClickHouse spells all three natively. `ClickHouseTypeSupport` is split into `protocols/type.py`, `ColumnTypeSupport` is declared in the protocol-conformance list, and `feature/backend/test_expression_roundtrip_all.py` now enumerates both core's and this backend's expression packages, classifying every outcome instead of swallowing it: the 31 dispatch gaps are grouped by feature in `UNMODELLED_FORMATTERS` and pinned in both directions, so a formatter nobody listed fails the run and a listed method no class needs fails too. diff --git a/changelog.d/5.changed.md b/changelog.d/5.changed.md new file mode 100644 index 0000000..ef55a4c --- /dev/null +++ b/changelog.d/5.changed.md @@ -0,0 +1 @@ +Render `TRIM`, `LPAD`, `RPAD` and `REPEAT` through the node formatters, and render the `ALTER COLUMN` default change ClickHouse actually accepts: `SET DEFAULT` becomes `DEFAULT` and `DROP DEFAULT` becomes `MODIFY COLUMN ... REMOVE DEFAULT`, both spellings quoted from the server's own rejection messages in `mixins/ddl_column.py`. `SHOW [FULL] COLUMNS`, `SHOW INDEX`, `SHOW PROCESSLIST`, `SHOW ENGINES` and `SHOW GRANTS` no longer refuse — five statements the server does accept that this backend used to raise on. Their columns are ClickHouse's, and the differences from MySQL's (where a parser reading MySQL's names returns `None` rather than raising) are documented rather than left for a caller to discover. diff --git a/changelog.d/5.docs.md b/changelog.d/5.docs.md new file mode 100644 index 0000000..eddc93a --- /dev/null +++ b/changelog.d/5.docs.md @@ -0,0 +1 @@ +Correct the capability-boundary documentation against measurements taken on ClickHouse 26.7.3.19, in both `docs/en_US` and `docs/zh_CN`: the `text` inverted index (GA from 26.2) replacing `tokenbf_v1` / `ngrambf_v1` for full-text search, the seven real `ST_*` WKB/MVT codecs and the `Geometry` / `Point` column types, the optimizer-hint comment the server parses and discards, the skip-index keyword being `TYPE` rather than MySQL's `USING`, and `FINAL` being accepted only by the engines that collapse duplicates. The README also records why `requires-python` is `>=3.10,<3.15`: the ceiling is inherited from `clickhouse-connect` (which declares `Requires-Python: <3.15,>=3.10`) rather than chosen here, which is why there is no Python 3.15 leg in CI. diff --git a/changelog.d/5.fixed.md b/changelog.d/5.fixed.md new file mode 100644 index 0000000..cf28358 --- /dev/null +++ b/changelog.d/5.fixed.md @@ -0,0 +1 @@ +Refuse the constraints whose silent outcome is worse than the error. `UNIQUE (col)` is a `SYNTAX_ERROR` on the server, and an unnamed `FOREIGN KEY (...) REFERENCES ...` is *accepted and silently discarded* — measured on 26.7.3.19, the DDL succeeds, `SHOW CREATE TABLE` reports no foreign key, `system.tables.create_table_query` does not contain the string, an orphan row inserts and reads back fine, and dropping the referenced parent succeeds as well. Emitting either would produce a schema with no constraint and nothing in the server's metadata to explain the missing enforcement, so `mixins/ddl_column.py` raises `UnsupportedFeatureError` at the point of declaration with that measured reason attached. diff --git a/changelog.d/5.internal.md b/changelog.d/5.internal.md new file mode 100644 index 0000000..31b218a --- /dev/null +++ b/changelog.d/5.internal.md @@ -0,0 +1 @@ +Adapt the tests and CI to core `release/v1.0.0.dev30`. Tests that read SQL types move from `column_type()` / `columns_type()` to `column_data_type()` / `columns_data_type()`; predicates the bare `Column` used to accept are built explicitly as `ComparisonPredicate` / `InPredicate` / `BinaryArithmeticExpression`; `DateTimeSupport` becomes `TimestampSupport` in the protocol-conformance list, `UUIDSupport` joins the not-implemented list, and `ClickHouseVectorSupport` leaves it. CI checks out core from `release/v1.0.0.dev30` now that the typed-columns branch has merged ([python-activerecord#126](https://github.com/rhosocial/python-activerecord/pull/126)), while the testsuite stays on `test/typed-column-contracts` because that branch has not landed on its release branch yet. `requirements-dev.txt` pins pydantic 2.12.2 / pydantic-core 2.41.4 for Python 3.13, since an unpinned resolve there picks a pydantic-core whose only cp313t distribution is an sdist. diff --git a/changelog.d/5.removed.md b/changelog.d/5.removed.md new file mode 100644 index 0000000..f6864aa --- /dev/null +++ b/changelog.d/5.removed.md @@ -0,0 +1 @@ +Remove the MySQL 9.0 `VECTOR` emulation surface and the MySQL declarative-partitioning strategy family. ClickHouse has no `STRING_TO_VECTOR` / `VECTOR_TO_STRING` / `VECTOR_DIM` / `DISTANCE_*` API, so `expression/vector.py`, `mixins/vector.py`, `protocols/vector.py`, `ClickHouseVectorMixin`, `ClickHouseVectorSupport` and the testsuite provider `tests/providers/partition.py` are gone; ClickHouse's own vector type is `QBit`, rendered by `ClickHouseVectorType` → `QBit(Float32, n)`. The `PARTITION ... VALUES` strategy family, subpartitioning, and `ADD` / `TRUNCATE` / `REORGANIZE` / `EXCHANGE` / `COALESCE` / `ANALYZE` / `CHECK` / `REBUILD` / `REPAIR PARTITION` are removed too, along with `expression/partition_lifecycle.py`. What is left is ClickHouse's own maintenance by partition id (`DROP` / `DETACH` / `ATTACH PARTITION ID`) and `system.parts` introspection, both now covered by `feature/backend/ddl/test_partition_maintenance.py`. diff --git a/docs/en_US/capabilities/unsupported.md b/docs/en_US/capabilities/unsupported.md index d28765a..e1b7145 100644 --- a/docs/en_US/capabilities/unsupported.md +++ b/docs/en_US/capabilities/unsupported.md @@ -7,8 +7,8 @@ This backend always fails fast (`UnsupportedFeatureError`) on features ClickHous | Feature | Behavior | Alternative | |------|------|------| | ACID cross-statement transactions | `transaction()` is a no-op; rollback raises | design per-part atomic mutations | -| `FOREIGN KEY` | not generated in DDL | maintain referential integrity at the application layer | -| `UNIQUE` constraint | not generated in DDL | `ReplacingMergeTree` dedup | +| `FOREIGN KEY` | not generated in DDL — **and note the server accepts one spelling without effect; see below** | maintain referential integrity at the application layer | +| `UNIQUE` constraint | not generated in DDL, and the server rejects it outright | `ReplacingMergeTree` dedup | | Triggers | raise | — | | Sequences | raise | client-side snowflake id | @@ -24,22 +24,56 @@ This backend always fails fast (`UnsupportedFeatureError`) on features ClickHous | Feature | Behavior | Alternative | |------|------|------| -| `FULLTEXT` index | raise | skip index `tokenbf_v1` + `hasToken` | +| `FULLTEXT` index | raise | `text` inverted index + `hasAllTokens`/`hasAnyTokens` | | `MATCH ... AGAINST` | raise | same as above | | `JSON_TABLE` | raise | `JSONExtract*` / `arrayJoin` | | SQL-standard spatial indexes | raise | skip indexes | +ClickHouse's own full-text facility is the `text` inverted index (GA from 26.2). +It is a skip index, so it is declared through `storage_options` like any other: + +```sql +CREATE TABLE docs ( + body String, + INDEX idx_body body TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 4 +) ENGINE = MergeTree ORDER BY tuple(); + +SELECT * FROM docs WHERE hasAllTokens(body, ['clickhouse', 'index']); +``` + +Two details worth knowing, both measured on 26.7.3.19 and both recorded in +: + +- `tokenbf_v1` and `ngrambf_v1` are **deprecated for full-text search from + 26.2**. The server still accepts them — `CREATE TABLE ... INDEX i body TYPE + tokenbf_v1(32768, 3, 0)` succeeds and `hasToken(body, 'beta')` still matches — + so an existing table keeps working, but new work should use `text`. +- `splitByNonAlpha` is case-sensitive. Add `preprocessor = lower()` for + case-insensitive matching, otherwise `hasAnyTokens(body, ['fox'])` misses + `'Fox'`. + ## Types | Feature | Behavior | Alternative | |------|------|------| -| Spatial types (`GEOMETRY`/`POINT`/...) | raise | store WKT as `String` | -| `ST_*` function family | raise | ClickHouse functions | -| MySQL 9.0 `VECTOR` type | raise | `Array(Float32)` | -| `STRING_TO_VECTOR`/`VECTOR_TO_STRING`/`VECTOR_DIM`/`DISTANCE_*` | raise | `L2Distance`/`cosineDistance`/`dotProduct` | +| Spatial types (`GEOMETRY`/`POINT`/...) | raise | a `Geometry` / `Point` column, or WKT as `String` | +| `ST_*` function family | raise | the 7 `ST_*` names ClickHouse has (WKB/MVT codecs) plus `geoDistance` | +| MySQL 9.0 `VECTOR` type | not offered — the API is absent | `ClickHouseVectorType` → `QBit(Float32, n)`, or `Array(Float32)` | +| `STRING_TO_VECTOR`/`VECTOR_TO_STRING`/`VECTOR_DIM`/`DISTANCE_*` | not offered — the API is absent | `L2Distance`/`cosineDistance`/`dotProduct` | | `SET` type | raise | `Enum16` or `Array(String)` | | `FIND_IN_SET` | raise | `has()`/`indexOf()` | +On the two spatial rows, measured on 26.7.3.19: ClickHouse **does** have +`Geometry` and `Point` column types (`CREATE TABLE t (g Geometry) ENGINE = +MergeTree` is accepted) and exactly seven `ST_*` functions — `ST_AsMVT`, +`ST_AsMVTGeom`, `ST_LineFromWKB`, `ST_MLineFromWKB`, `ST_PolyFromWKB`, +`ST_MPolyFromWKB`, `ST_PointFromWKB` — every one of them a WKB / MVT codec. +What is absent is the OGC surface this backend's `ST_*` mixin would emit: +`ST_Distance` / `ST_AsText` answer `UNKNOWN_FUNCTION`, and WKT parsing for the +`Geometry` type answers `Cannot parse text value of type Geometry here: +POINT(1 2)`. `geoDistance(lon1, lat1, lon2, lat2)` is the distance helper that +does exist. + ## Statements | Feature | Behavior | Alternative | @@ -48,21 +82,146 @@ This backend always fails fast (`UnsupportedFeatureError`) on features ClickHous | `LOAD DATA INFILE` | raise | format parsers / `input()` table function | | `LOAD XML` | raise | same as above | | `TABLE` / `VALUES` table-value constructor (MySQL 8.0.19+) | raise | `SELECT ... UNION ALL SELECT ...` | -| Whole-table maintenance (`ANALYZE`/`CHECK`/`CHECKSUM`/`REPAIR TABLE`) | raise | `OPTIMIZE TABLE ... FINAL` or `SYSTEM` commands | -| MySQL optimizer hints (`/*+ SET_VAR */`) | raise | `SETTINGS` clause | +| Whole-table maintenance (`ANALYZE`/`CHECKSUM`/`REPAIR TABLE`) | raise. `CHECK TABLE` is the exception and *does* work — see below | `OPTIMIZE TABLE ... FINAL` or `SYSTEM` commands | +| MySQL optimizer hints (`/*+ SET_VAR */`) | raise, and the refusal is not redundant: the server **parses the comment and ignores its contents** | `SETTINGS` clause | | JSON Relational Duality Views | raise | `JSON` type + `JSONExtract*` | +| `SHOW TABLE STATUS` / `SHOW TRIGGERS` / `SHOW CREATE TRIGGER` / `SHOW WARNINGS` / `SHOW ERRORS` / `SHOW VARIABLES` / `SHOW STATUS` / `SHOW CHARACTER SET` / `SHOW COLLATION` / `SHOW PLUGINS` | raise | the `system.*` tables each names, via `ClickHouseStatusIntrospector` | +| `SHOW [FULL] COLUMNS` / `SHOW INDEX` / `SHOW PROCESSLIST` / `SHOW ENGINES` / `SHOW GRANTS` | **not refused — these work** and are read via `backend.introspector.show.*()` | `DESCRIBE TABLE` / `system.columns`, `system.tables` + `system.data_skipping_indices`, `system.processes`, `system.table_engines`, `system.grants` if you prefer the system tables directly | + +### The `SHOW` set ClickHouse actually has + +Five statements in the MySQL `SHOW` set that this backend used to refuse all work +on the server, and are now rendered and parsed. Each reads the column names the +server actually sends, which differ from MySQL's in ways that fail silently +rather than loudly — a parser looking for MySQL's names returns `None` fields, not +an exception: + +| statement | ClickHouse columns | differs from MySQL | +|-----------|-------------------|--------------------| +| `SHOW [FULL] COLUMNS FROM db.t` | `field`, `type`, `null`, `key`, `default`, `extra` (+ `collation`, `comment`, `privileges` under `FULL`) | `key` is `PRI SOR` / `PRI` / `''`, not `PRI`/`UNI`/`MUL`; `default` carries the expression of an `ALIAS`, `DEFAULT` **or** `MATERIALIZED` column; rows come back in **name** order, not declaration order | +| `SHOW INDEX FROM db.t` | `table`, `non_unique`, `key_name`, `seq_in_index`, `pk_col`, `collation`, `cardinality`, `sub_part`, `packed`, `null`, `index_type`, `comment`, `index_comment`, `visible`, `expression` | the indexed column is `pk_col`, not `Column_name`; `index_type` is `PRIMARY`/`MINMAX`/`BLOOM_FILTER`/…, not `BTREE` | +| `SHOW PROCESSLIST` | all 43 of `system.processes`' columns | lower-cased; there is no per-process `Command` or `State`, so those result fields are always `None`; and there is **no `FULL` variant** (`SHOW FULL PROCESSLIST` is a `SYNTAX_ERROR`) | +| `SHOW ENGINES` | `name`, eight `supports_*` flags, `description`, `syntax`, `examples`, `introduced_in`, `related` | none of MySQL's `Support`/`Transactions`/`XA`/`Savepoints` | +| `SHOW GRANTS [FOR user]` | **one column whose name is the statement text plus the output format** — `GRANTS`, `GRANTS FORMAT JSONEachRow`, `GRANTS FORMAT Native`, `GRANTS FOR root` — never MySQL's `Grants for` | so the single column's *value* is read, not a named key. `FOR user@host` is also not ClickHouse's: an access entity is a role name, and `SHOW GRANTS FOR root@localhost` is `UNKNOWN_ROLE` | + +Two further statements remain refusals because the server really does not have +them: `SHOW VARIABLES` and `SHOW STATUS` — the server's grammar after `SHOW` +offers no such keyword, and the configuration lives in `system.settings` / +`system.server_settings` while runtime counters live in `system.metrics`, +`system.events` and `system.asynchronous_metrics`, all read by +`ClickHouseStatusIntrospector`. Likewise `SHOW TABLE STATUS`, `SHOW TRIGGERS`, +`SHOW CREATE TRIGGER`, `SHOW WARNINGS`, `SHOW ERRORS`, `SHOW CHARACTER SET`, +`SHOW COLLATION` and `SHOW PLUGINS`. + +The inventory that proves those absences is the server's own grammar. On +26.7.3.19 the complete set of keywords `SHOW` accepts is `CREATE`, `FULL`, +`DATABASES`, `CLUSTERS`, `MERGES`, `FILESYSTEM CACHES`, `CLUSTER`, `CHANGED`, +`SETTINGS`, `TEMPORARY`, `TABLES`, `DICTIONARIES`, `EXTENDED`, `COLUMNS`, +`FIELDS`, `ENGINES`, `FUNCTIONS`, `INDEX`, `INDEXES`, `INDICES`, `KEYS`, +`SETTING`, `DATABASE`, `VIEW`, `PROCESSLIST`, `ACCESS`, `USERS`, `ROLES`, +`PROFILES`, `POLICIES`, `QUOTAS`, `MASKING POLICIES`, `CURRENT ROLES`, +`ENABLED ROLES`, `CURRENT QUOTA`, `QUOTA`, `GRANTS`, `PRIVILEGES` — no +`VARIABLES`, no `STATUS`, no `TRIGGERS`, no `PLUGINS`. See also +. + +### FOREIGN KEY is accepted without effect + +`FOREIGN KEY` is worth calling out because the server does **not** simply reject +it. Three of the four spellings are rejected outright, and the fourth — the one a +generic SQL writer is most likely to produce — is **accepted, silently discarded, +and has no effect whatsoever**. All measured on 26.7.3.19: + +| spelling | result | +|----------|--------| +| `CREATE TABLE c (id UInt32, pid UInt32, FOREIGN KEY (pid) REFERENCES p(id)) ENGINE = MergeTree ORDER BY id` | **succeeds.** `SHOW CREATE TABLE c` reports no foreign key, `system.tables.create_table_query` does not contain the string `FOREIGN`, and an orphan row inserts and reads back fine | +| `CREATE TABLE c (id UInt32, CONSTRAINT fk FOREIGN KEY (id) REFERENCES p(id)) ...` | `Code: 62. Syntax error: failed at position 57 (FOREIGN) ... Expected one of: CHECK, ASSUME. (SYNTAX_ERROR)` | +| `CREATE TABLE c (id UInt32 REFERENCES p(id)) ...` (inline column reference) | `Code: 62. Syntax error: failed at position 43 (REFERENCES) ...` | +| `CREATE TABLE c (id UInt32) ENGINE = MergeTree ORDER BY id FOREIGN KEY (id) REFERENCES p(id)` (after the storage clauses) | `Code: 62. Syntax error: failed at position 75 (FOREIGN) ...` | + +Dropping the referenced parent table also succeeds, and the orphan row is still +readable — so the clause is not a deferred check either, it is gone. + +Two consequences for this backend: + +* **Emitting it would be the worst of the three outcomes.** It would not raise, + so the failure would surface later as missing referential integrity, in + production data, with nothing in the server's metadata to explain it. That is + precisely the "silent emulation" this backend's + [design principle](../introduction/capability_boundaries.md) exists to avoid, + and it is why `format_table_constraint` raises for `FOREIGN KEY` rather than + rendering the clause that happens to parse. +* **"Not generated in DDL" is not the same claim as "the server rejects it."** A + reader who assumed the second would test it by pasting `FOREIGN KEY` into a + `CREATE TABLE` and, for the unnamed spelling, conclude the server supports it. + +For contrast, the constraint forms ClickHouse *does* have are `CHECK` and +`ASSUME` (both documented at +, and +`CHECK` is enforced — an out-of-range `UInt32` is stored wrapped rather than +rejected only because the comparison is done on the stored value). And `UNIQUE +(col)` is rejected outright: `Code: 62. Syntax error: failed at position 50 +(() ... Expected one of: NULL, NOT, DEFAULT, MATERIALIZED, EPHEMERAL, ALIAS, +AUTO_INCREMENT, TTL, PRIMARY KEY, data type, identifier. (SYNTAX_ERROR)`. + +### CHECK TABLE works + +`CHECK TABLE` is a real ClickHouse statement and this backend now emits it; it +is not in the fail-fast list above. It dispatches on the table engine, so what it +verifies depends on the engine. + +## Partitioning + +ClickHouse *does* partition, but not the way MySQL does: `PARTITION BY ` on a +`MergeTree` table, with no strategy keyword, no `VALUES LESS THAN` / +`VALUES IN` / `MAXVALUE` boundary and no subpartitioning. The clause is declared +through `storage_options`; maintenance is by partition id +(`DROP` / `DETACH` / `ATTACH PARTITION ID`); introspection is `system.parts`. +The MySQL statements below have no ClickHouse spelling and are absent rather +than switched off one at a time — the inventory that proves the absence is +. + +| Feature | Behavior | Alternative | +|------|------|------| +| Declarative `PARTITION BY RANGE/LIST/HASH/KEY` with `PARTITION ... VALUES ...` | raise | `PARTITION BY ` in `storage_options` | +| `ADD PARTITION` | raise | a partition appears when a row lands in it | +| `TRUNCATE PARTITION` | raise | `ALTER TABLE ... DELETE IN PARTITION ... WHERE ...`, or `TRUNCATE TABLE` | +| `REORGANIZE` / `EXCHANGE` / `COALESCE PARTITION`, `REMOVE PARTITIONING` | raise | `REPLACE PARTITION` / `MOVE PARTITION TO TABLE` | +| `ANALYZE`/`CHECK`/`OPTIMIZE`/`REBUILD`/`REPAIR PARTITION` | raise | `SYSTEM RESTART`/`SYSTEM RELOAD`, `OPTIMIZE TABLE ... PARTITION` | +| `SUBPARTITION BY` | raise | a tuple partition key (`PARTITION BY (a, b)`) | +| `MAXVALUE` partition boundary | raise | the partition-key value itself | +| `information_schema.PARTITIONS` | raise | `system.parts` | ## Admin commands (MySQL admin set) -`FLUSH`/`RESET`/`CACHE INDEX`/`LOAD INDEX INTO CACHE`/`INSTALL`/`UNINSTALL COMPONENT`/`PLUGIN`/`CLONE`/`RESTART`/`BINLOG`/`HANDLER`/`DO`/`KILL`/`SHUTDOWN`/`HELP`/`CREATE USER`/`DROP USER`/`GRANT`/`REVOKE` — all raise. Use ClickHouse's `SYSTEM` command family instead (`SYSTEM RELOAD`/`SYSTEM KILL`/`SYSTEM FLUSH`/...). +`FLUSH`/`RESET`/`CACHE INDEX`/`LOAD INDEX INTO CACHE`/`INSTALL`/`UNINSTALL COMPONENT`/`PLUGIN`/`CLONE`/`RESTART`/`BINLOG`/`HANDLER`/`DO`/`KILL`/`SHUTDOWN`/`HELP` — all raise. Use ClickHouse's `SYSTEM` command family instead (`SYSTEM RELOAD`/`SYSTEM KILL`/`SYSTEM FLUSH`/...). `KILL` is not a single MySQL-shaped statement: ClickHouse's is `KILL QUERY|MUTATION|PART_MOVE_TO_SHARD|TRANSACTION ...`, and `KILL 1` answers "Expected one of: QUERY, MUTATION, PART_MOVE_TO_SHARD, TRANSACTION". + +Four names on that list used to sit here and are not MySQL-only at all: +`CREATE USER`, `ALTER USER`, `DROP USER`, `GRANT` and `REVOKE` are ClickHouse's +own access-control statements and all succeed on 26.7.3.19 — the backend simply +offers no core API that emits them, so nothing in this backend routes a +`CREATE USER` to the server. Access management from the SQL console is not +blocked; only this backend's expression layer has no spelling for it. ## Async | Feature | Behavior | Alternative | |------|------|------| -| Async backend | instantiating `AsyncClickHouseBackend` immediately raises `NotImplementedError` | sync backend + out-of-process concurrency | - -`clickhouse-connect` is a purely synchronous library; this backend provides no async implementation. +| `AsyncClickHouseBackend` | instantiating it raises `NotImplementedError` | `ClickHouseBackend` (sync) + out-of-process concurrency | + +The narrow claim is the true one: **there is no async backend class you can +build a connection with.** `AsyncClickHouseBackend(connection_config=...)` raises +`NotImplementedError` from `__init__`, because `clickhouse-connect` is a +synchronous-only driver, and `AsyncClickHouseTransactionManager` raises the same +way. + +It is not true that this package ships no async code. It ships two complete +async introspectors — `AsyncClickHouseStatusIntrospector` and +`AsyncShowIntrospector` — each implementing every method of core's async status / +SHOW contracts with real `await`ed queries, none of which raises +`NotImplementedError`. They are reachable by any caller that supplies its own +async backend object (an async ClickHouse driver, or a wrapper); only the stock +`AsyncClickHouseBackend` refuses. The async test fixtures skip for that same +single reason: they have no instantiable backend to hand them. ## How to probe diff --git a/docs/en_US/ddl/skip_indexes.md b/docs/en_US/ddl/skip_indexes.md index 59e6432..ee6681e 100644 --- a/docs/en_US/ddl/skip_indexes.md +++ b/docs/en_US/ddl/skip_indexes.md @@ -13,7 +13,7 @@ CREATE TABLE events ( ) ENGINE = MergeTree ORDER BY (ts) ``` -`INDEX ... USING type` is supported at this backend's dialect layer. +The index-type keyword is `TYPE`, not MySQL's `USING`: `INDEX i (s) TYPE minmax GRANULARITY 4` is accepted, while `INDEX i (s) USING minmax` is a `SYNTAX_ERROR` on 26.7.3.19. ## Common skip index types @@ -23,22 +23,44 @@ CREATE TABLE events ( | `set(max_rows)` | stores the set of distinct granule values | low cardinality | | `bloom_filter` | Bloom filter, judge possible existence | any equality | | `bloom_filter(bf)` | parameterized Bloom | same as above | -| `tokenbf_v1` | tokenizing Bloom | text token search | -| `ngrambf_v1(n, sz, b, f)` | n-gram Bloom | substring search | +| `text(tokenizer = ...)` | inverted index over tokenized text | full-text search | +| `ngrambf_v1(n, sz, b, f)` | n-gram Bloom (deprecated for full text from 26.2) | substring search | +| `tokenbf_v1(sz, f, seed)` | tokenizing Bloom (deprecated for full text from 26.2) | word search | + +The server's own list of what it accepts is in the error text: declaring an +unknown type answers ``Unknown Index type 'fulltext'. Available index types: +hypothesis, text, vector_similarity, bloom_filter, sparse_grams, tokenbf_v1, +ngrambf_v1, set, minmax``. ## Substitute for FULLTEXT -ClickHouse does **not** support MySQL's `FULLTEXT` index and `MATCH...AGAINST` (this backend fails fast). Use `tokenbf_v1` + `hasToken` for text search: +ClickHouse does **not** support MySQL's `FULLTEXT` index and `MATCH...AGAINST` +(this backend fails fast). Its full-text facility is the `text` inverted index, +GA from 26.2: ```sql CREATE TABLE docs ( body String, - INDEX idx_body body TYPE tokenbf_v1(3, 256, 2, 0) GRANULARITY 4 -) ENGINE = MergeTree ORDER BY id; + INDEX idx_body body TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 4 +) ENGINE = MergeTree ORDER BY tuple(); -SELECT * FROM docs WHERE hasToken(body, 'clickhouse'); +SELECT * FROM docs WHERE hasAllTokens(body, ['clickhouse', 'index']); +SELECT * FROM docs WHERE hasAnyTokens(body, ['index', 'inverted']); ``` +`GRANULARITY` is accepted but ignored for a text index — the index is built for +the whole part and `SHOW CREATE TABLE` reports `GRANULARITY 100000000`. + +`tokenbf_v1` + `hasToken` still works on 26.7.3.19 (`CREATE TABLE ... TYPE +tokenbf_v1(32768, 3, 0)` is accepted and `hasToken(body, 'beta')` matches), but +ClickHouse's own documentation records `tokenbf_v1` and `ngrambf_v1` as +deprecated for full-text search from 26.2 in favour of `text`, so prefer `text` +for new tables. See +. + +`splitByNonAlpha` is case-sensitive: add `preprocessor = lower(body)` to the +index for case-insensitive matching. + ## Vector approximate nearest neighbor (substitute for MySQL VECTOR index) Use a `vector_similarity` skip index (with an `Array(Float32)` column) for ANN search, instead of MySQL's `VECTOR` type and `DISTANCE_*` functions. diff --git a/docs/en_US/introduction/capability_boundaries.md b/docs/en_US/introduction/capability_boundaries.md index 38a2332..3909f79 100644 --- a/docs/en_US/introduction/capability_boundaries.md +++ b/docs/en_US/introduction/capability_boundaries.md @@ -11,7 +11,9 @@ raises `UnsupportedFeatureError`, **instead of**: - silently ignoring (e.g. making `transaction()` a no-op while claiming it committed); - degrading into an inefficient emulation (e.g. simulating UPSERT with `SELECT + DELETE + INSERT`); -- generating SQL ClickHouse would reject (e.g. `FOREIGN KEY`). +- generating SQL ClickHouse would reject (e.g. `UNIQUE (col)`), or SQL it + would accept and silently ignore (e.g. an unnamed `FOREIGN KEY (...)` — + see [FOREIGN KEY is accepted without effect](../capabilities/unsupported.md#foreign-key-is-accepted-without-effect)). The only exception is `transaction()`: it degrades to a **no-op context manager** so generic code paths (core library and testsuite transaction-related tests) keep @@ -36,14 +38,27 @@ except UnsupportedFeatureError as e: - Columnar types: `Int*`/`UInt*`, `Float*`, `Decimal*`, `String`/`FixedString`, `Date`/`Date32`/`DateTime`/`DateTime64`, `Bool`, `UUID`, `IPv4`/`IPv6`, - `Enum8`/`Enum16`, `Array`, `Map`, `Tuple`, `Nullable(T)`, `LowCardinality(T)`, `JSON` + `Enum8`/`Enum16`, `Array`, `Map`, `Tuple`, `Nullable(T)`, `LowCardinality(T)`, `JSON`, + and (as of the 26.x line) `Geometry` / `Point` - DDL: `ENGINE`, `ORDER BY` sorting key, `PARTITION BY`, `TTL`, skip indexes - (`INDEX ... USING`) -- Querying: CTE (`WITH`), window functions, `QUALIFY`, `FINAL`, `ARRAY JOIN`, set - operations (`UNION`/`INTERSECT`/`EXCEPT`, explicit `ALL`/`DISTINCT`), `EXPLAIN` -- JSON: `JSONExtractString`/`JSONExtractRaw`/`JSON_VALUE` and the native function - family (**not** MySQL arrow operators `->`/`->>`) -- Introspection: `system.*` tables (settings/metrics/replicas/processes/...), `SHOW` + (`INDEX name (col) TYPE GRANULARITY n` — the keyword is `TYPE`, not + MySQL's `USING`) +- Partition maintenance by partition id: `DROP` / `DETACH` / `ATTACH + PARTITION ID` (ClickHouse's own clause list is at + ) +- Querying: CTE (`WITH`), window functions, `QUALIFY`, `ARRAY JOIN`, set + operations (`UNION`/`INTERSECT`/`EXCEPT`, explicit `ALL`/`DISTINCT`), `EXPLAIN`. + `FINAL` is a query modifier too, but only for the engines that collapse + duplicates — a plain `MergeTree` answers `Storage MergeTree doesn't support + FINAL`, while `ReplacingMergeTree` accepts it. +- JSON: `JSONExtractString`/`JSONExtractRaw`/`JSON_VALUE` (on a `String` column — + `JSON_VALUE` rejects a `JSON` argument) and the native function family + (**not** MySQL arrow operators `->`/`->>`) +- Introspection: `system.*` tables (settings/server_settings/metrics/events/ + asynchronous_metrics/replicas/processes/...), `SHOW` +- Access control from SQL: `CREATE USER` / `ALTER USER` / `DROP USER` / + `GRANT` / `REVOKE` (this backend's expression layer offers no spelling for + them, so nothing here emits them — but the server has them) - Client-side snowflake `Int64` IDs (replacing `AUTO_INCREMENT`) - Lightweight UPDATE/DELETE mutations (`enable_block_number_column`/ `enable_block_offset_column` settings required — see [Mutations](../capabilities/mutations.md)) @@ -53,23 +68,26 @@ except UnsupportedFeatureError as e: | Capability | Behavior | Alternative | |------------|----------|-------------| | ACID cross-statement transactions | `transaction()` no-op; rollback raises | design per per-part atomic mutations, or use `ReplacingMergeTree` | -| `FOREIGN KEY` / `UNIQUE` constraints | DDL not emitted | dedup via table engine | +| `FOREIGN KEY` / `UNIQUE` constraints | DDL not emitted. `UNIQUE (col)` is a `SYNTAX_ERROR` on the server; an unnamed `FOREIGN KEY (...) REFERENCES ...` is **accepted and silently discarded** — see [FOREIGN KEY is accepted without effect](../capabilities/unsupported.md#foreign-key-is-accepted-without-effect) | dedup via table engine; referential integrity at the application layer | | Triggers, sequences | raise | — | | UPSERT / `ON CONFLICT` / `INSERT IGNORE` / `REPLACE INTO` | raise | `ReplacingMergeTree` + explicit `INSERT` | | `FOR UPDATE` row locking | raise | OLAP does not do pessimistic locking | -| `FULLTEXT` indexes / `MATCH...AGAINST` | raise | skip indexes (`tokenbf_v1` + `hasToken`) | +| `FULLTEXT` indexes / `MATCH...AGAINST` | raise | the `text` inverted index (GA from 26.2) + `hasAllTokens`/`hasAnyTokens` | | `JSON_TABLE` | raise | `JSONExtract*` / `arrayJoin` | -| Spatial types (`GEOMETRY`/`POINT`/...) and `ST_*` functions | raise | store WKT as `String`, compute with ClickHouse functions | -| MySQL 9.0 `VECTOR` type and `STRING_TO_VECTOR`/`VECTOR_DIM`/`DISTANCE_*` | raise | `Array(Float32)` + `L2Distance`/`cosineDistance` | +| Spatial types (`GEOMETRY`/`POINT`/...) and the OGC `ST_*` functions | raise | ClickHouse's own `Geometry`/`Point` column types and its 7 WKB/MVT `ST_*` codecs; `geoDistance` for distance | +| MySQL 9.0 `VECTOR` type and `STRING_TO_VECTOR`/`VECTOR_DIM`/`DISTANCE_*` | not offered — the API is absent (ClickHouse's own vector type is `QBit`, rendered by `ClickHouseVectorType`) | `QBit(Float32, n)` or `Array(Float32)` + `L2Distance`/`cosineDistance` | | `SET` type / `FIND_IN_SET` | raise | `Enum16` or `Array(String)` + `has()` | | Stored procedures / functions / `CALL` | raise | ClickHouse SQL UDFs (`CREATE FUNCTION ... AS`) | | `LOAD DATA INFILE` / `LOAD XML` | raise | format parsers / `input()` table function | | MySQL admin commands (`FLUSH`/`RESET`/`KILL`/`INSTALL PLUGIN`/`CLONE`/`BINLOG`/`HANDLER`/`GRANT`/`CREATE USER`) | raise | ClickHouse `SYSTEM` command family | | `TABLE` / `VALUES` table-value constructor (MySQL 8.0.19+) | raise | `SELECT ... UNION ALL SELECT ...` | -| Whole-table maintenance (`ANALYZE`/`CHECK`/`CHECKSUM`/`REPAIR TABLE`) | raise | `OPTIMIZE TABLE ... FINAL` or `SYSTEM` commands | -| MySQL optimizer hints (`/*+ SET_VAR */`) | raise | `SETTINGS` clause | +| Whole-table maintenance (`ANALYZE`/`CHECKSUM`/`REPAIR TABLE`) | raise (`CHECK TABLE` is the exception and *does* work) | `OPTIMIZE TABLE ... FINAL` or `SYSTEM` commands | +| MySQL optimizer hints (`/*+ SET_VAR */`) | raise. The refusal is not redundant: ClickHouse **parses the comment and discards its contents**, measured on 26.7.3.19 — `/*+ SET_VAR(max_block_size=7) */` leaves `getSetting('max_block_size')` at `65409`, and a hint naming an invented setting (`SET_VAR(totally_bogus_zzz=1)`) is also accepted, so the name is not validated either | `SETTINGS` clause | | JSON Relational Duality Views | raise | `JSON` type + `JSONExtract*` | -| Async backend | `AsyncClickHouseBackend` placeholder raises `NotImplementedError` on instantiation | sync backend + out-of-process concurrency | +| `SHOW` statements ClickHouse does not have (`SHOW VARIABLES`/`STATUS`/`TABLE STATUS`/`TRIGGERS`/`CREATE TRIGGER`/`WARNINGS`/`ERRORS`/`CHARACTER SET`/`COLLATION`/`PLUGINS`) | raise | the `system.*` tables each names, via `ClickHouseStatusIntrospector`. `SHOW` accepts no `VARIABLES`, `STATUS`, `TRIGGERS` or `PLUGINS` keyword at all | +| `SHOW` statements ClickHouse **does** have (`SHOW [FULL] COLUMNS`/`INDEX`/`PROCESSLIST`/`ENGINES`/`GRANTS`) | **not refused — these work** and are read via `backend.introspector.show.*()`. Their columns are ClickHouse's, so a parser reading MySQL's names gets `None` fields rather than an error | the `system.*` tables, if you prefer to query them directly. See [The `SHOW` set ClickHouse actually has](../capabilities/unsupported.md#the-show-set-clickhouse-actually-has) | +| Declarative partitioning (`PARTITION BY RANGE/LIST/HASH/KEY` with `PARTITION ... VALUES ...`, subpartitioning, `ADD`/`TRUNCATE`/`REORGANIZE`/`EXCHANGE`/`COALESCE`/`ANALYZE`/`CHECK`/`REBUILD`/`REPAIR PARTITION`) | raise — no such clause exists; see [Partitioning](../capabilities/unsupported.md#partitioning) | `PARTITION BY ` via `storage_options`; maintenance by `DROP`/`DETACH`/`ATTACH PARTITION ID`; introspection via `system.parts` | +| Async backend | there is no async backend to instantiate: `AsyncClickHouseBackend(...)` raises `NotImplementedError` | sync backend + out-of-process concurrency. The two async *introspectors* in this package (`AsyncClickHouseStatusIntrospector`, `AsyncShowIntrospector`) are complete and await real queries — they just have no stock backend to run against. | For the complete list, see [Unsupported features](../capabilities/unsupported.md). diff --git a/docs/zh_CN/capabilities/unsupported.md b/docs/zh_CN/capabilities/unsupported.md index 9f351ba..8bd4b97 100644 --- a/docs/zh_CN/capabilities/unsupported.md +++ b/docs/zh_CN/capabilities/unsupported.md @@ -7,8 +7,8 @@ | 功能 | 行为 | 替代 | |------|------|------| | ACID 跨语句事务 | `transaction()` 空操作;rollback 抛异常 | 按 per-part 原子变更设计 | -| `FOREIGN KEY` | DDL 不生成 | 应用层维护引用完整性 | -| `UNIQUE` 约束 | DDL 不生成 | `ReplacingMergeTree` 去重 | +| `FOREIGN KEY` | DDL 不生成——**注意:其中一种写法服务器会接受但不生效,详见下文** | 应用层维护引用完整性 | +| `UNIQUE` 约束 | DDL 不生成,且服务器直接拒绝 | `ReplacingMergeTree` 去重 | | 触发器 | raise | — | | 序列 | raise | 客户端雪花 id | @@ -24,22 +24,53 @@ | 功能 | 行为 | 替代 | |------|------|------| -| `FULLTEXT` 索引 | raise | skip index `tokenbf_v1` + `hasToken` | +| `FULLTEXT` 索引 | raise | `text` 倒排索引 + `hasAllTokens`/`hasAnyTokens` | | `MATCH ... AGAINST` | raise | 同上 | | `JSON_TABLE` | raise | `JSONExtract*` / `arrayJoin` | | SQL 标准空间索引 | raise | skip indexes | +ClickHouse 自己的全文设施是 `text` 倒排索引(26.2 起 GA)。它也是一种 skip +index,因此和其他 skip index 一样通过 `storage_options` 声明: + +```sql +CREATE TABLE docs ( + body String, + INDEX idx_body body TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 4 +) ENGINE = MergeTree ORDER BY tuple(); + +SELECT * FROM docs WHERE hasAllTokens(body, ['clickhouse', 'index']); +``` + +两个细节,均在 26.7.3.19 上实测,且都记录在 +: + +- `tokenbf_v1` 与 `ngrambf_v1` 自 **26.2** 起在全文检索用途上**已废弃**。服务端 + 仍然接受它们——`CREATE TABLE ... INDEX i body TYPE tokenbf_v1(32768, 3, 0)` + 成功,`hasToken(body, 'beta')` 也仍然命中——所以既有表继续可用,但新工作应改用 + `text`。 +- `splitByNonAlpha` 区分大小写。需要忽略大小写时加上 `preprocessor = lower()`, + 否则 `hasAnyTokens(body, ['fox'])` 会漏掉 `'Fox'`。 + ## 类型 | 功能 | 行为 | 替代 | |------|------|------| -| 空间类型(`GEOMETRY`/`POINT`/...)| raise | 存 WKT 为 `String` | -| `ST_*` 函数族 | raise | ClickHouse 函数 | -| MySQL 9.0 `VECTOR` 类型 | raise | `Array(Float32)` | -| `STRING_TO_VECTOR`/`VECTOR_TO_STRING`/`VECTOR_DIM`/`DISTANCE_*` | raise | `L2Distance`/`cosineDistance`/`dotProduct` | +| 空间类型(`GEOMETRY`/`POINT`/...)| raise | `Geometry` / `Point` 列类型,或把 WKT 存为 `String` | +| `ST_*` 函数族 | raise | ClickHouse 实际有的 7 个 `ST_*`(WKB/MVT 编解码)+ `geoDistance` | +| MySQL 9.0 `VECTOR` 类型 | 不提供——该 API 已移除 | `ClickHouseVectorType` → `QBit(Float32, n)`,或 `Array(Float32)` | +| `STRING_TO_VECTOR`/`VECTOR_TO_STRING`/`VECTOR_DIM`/`DISTANCE_*` | 不提供——该 API 已移除 | `L2Distance`/`cosineDistance`/`dotProduct` | | `SET` 类型 | raise | `Enum16` 或 `Array(String)` | | `FIND_IN_SET` | raise | `has()`/`indexOf()` | +关于这两行空间类型,在 26.7.3.19 上实测:ClickHouse **确实**有 `Geometry` 与 +`Point` 列类型(`CREATE TABLE t (g Geometry) ENGINE = MergeTree` 会被接受), +也**确实**有且仅有 7 个 `ST_*` 函数——`ST_AsMVT`、`ST_AsMVTGeom`、 +`ST_LineFromWKB`、`ST_MLineFromWKB`、`ST_PolyFromWKB`、`ST_MPolyFromWKB`、 +`ST_PointFromWKB`——全部是 WKB / MVT 编解码。缺的是本后端 `ST_*` mixin 会生成的 +OGC 函数面:`ST_Distance`/`ST_AsText` 返回 `UNKNOWN_FUNCTION`,`Geometry` 类型也 +不接受 WKT 文本(`Cannot parse text value of type Geometry here: POINT(1 2)`)。 +真正存在的距离函数是 `geoDistance(lon1, lat1, lon2, lat2)`。 + ## 语句 | 功能 | 行为 | 替代 | @@ -48,21 +79,132 @@ | `LOAD DATA INFILE` | raise | 格式解析器 / `input()` 表函数 | | `LOAD XML` | raise | 同上 | | `TABLE` / `VALUES` 表值构造(MySQL 8.0.19+)| raise | `SELECT ... UNION ALL SELECT ...` | -| 整表维护(`ANALYZE`/`CHECK`/`CHECKSUM`/`REPAIR TABLE`)| raise | `OPTIMIZE TABLE ... FINAL` 或 `SYSTEM` 命令 | -| MySQL optimizer hints(`/*+ SET_VAR */`)| raise | `SETTINGS` 子句 | +| 整表维护(`ANALYZE`/`CHECKSUM`/`REPAIR TABLE`)| raise。`CHECK TABLE` 是例外,它确实可用——见下文 | `OPTIMIZE TABLE ... FINAL` 或 `SYSTEM` 命令 | +| MySQL optimizer hints(`/*+ SET_VAR */`)| raise,且这个拒绝并非多余:服务器**会解析该注释但忽略其内容** | `SETTINGS` 子句 | | JSON Relational Duality Views | raise | `JSON` 类型 + `JSONExtract*` | +| `SHOW TABLE STATUS` / `SHOW TRIGGERS` / `SHOW CREATE TRIGGER` / `SHOW WARNINGS` / `SHOW ERRORS` / `SHOW VARIABLES` / `SHOW STATUS` / `SHOW CHARACTER SET` / `SHOW COLLATION` / `SHOW PLUGINS` | raise | 各自的 `system.*` 表,经 `ClickHouseStatusIntrospector` 读取 | +| `SHOW [FULL] COLUMNS` / `SHOW INDEX` / `SHOW PROCESSLIST` / `SHOW ENGINES` / `SHOW GRANTS` | **不再拒绝——这几条确实可用**,经 `backend.introspector.show.*()` 读取 | 若想直接查系统表:`DESCRIBE TABLE` / `system.columns`、`system.tables` + `system.data_skipping_indices`、`system.processes`、`system.table_engines`、`system.grants` | + +### ClickHouse 实际具备的 `SHOW` 集合 + +MySQL `SHOW` 集合中有五条语句本后端过去拒绝,但服务器上其实可用,现已渲染并解析。 +每条都按服务器真实返回的列名读取,而这些列名与 MySQL 的差异是**静默失败** +而非报错——按 MySQL 的列名去找只会拿到 `None`,不会抛异常: + +| 语句 | ClickHouse 列名 | 与 MySQL 的差异 | +|------|-----------------|------------------| +| `SHOW [FULL] COLUMNS FROM db.t` | `field`、`type`、`null`、`key`、`default`、`extra`(`FULL` 下另有 `collation`、`comment`、`privileges`)| `key` 取值是 `PRI SOR` / `PRI` / `''`,不是 `PRI`/`UNI`/`MUL`;`default` 同时承载 `ALIAS`、`DEFAULT` **与** `MATERIALIZED` 的表达式;行按**列名**排序,不是声明顺序 | +| `SHOW INDEX FROM db.t` | `table`、`non_unique`、`key_name`、`seq_in_index`、`pk_col`、`collation`、`cardinality`、`sub_part`、`packed`、`null`、`index_type`、`comment`、`index_comment`、`visible`、`expression` | 被索引的列叫 `pk_col`,不是 `Column_name`;`index_type` 是 `PRIMARY`/`MINMAX`/`BLOOM_FILTER`/…,不是 `BTREE` | +| `SHOW PROCESSLIST` | `system.processes` 的全部 43 列 | 全小写;没有每进程的 `Command` 或 `State`,因此结果里这两个字段恒为 `None`;且**没有 `FULL` 变体**(`SHOW FULL PROCESSLIST` 是 `SYNTAX_ERROR`)| +| `SHOW ENGINES` | `name`、八个 `supports_*` 标志、`description`、`syntax`、`examples`、`introduced_in`、`related` | MySQL 的 `Support`/`Transactions`/`XA`/`Savepoints` 一个都没有 | +| `SHOW GRANTS [FOR user]` | **只有一列,列名是语句文本加上输出格式**——`GRANTS`、`GRANTS FORMAT JSONEachRow`、`GRANTS FORMAT Native`、`GRANTS FOR root`——绝不是 MySQL 的 `Grants for` | 因此读的是该列的**值**而非具名键。`FOR user@host` 也不是 ClickHouse 的写法:访问实体是角色名,`SHOW GRANTS FOR root@localhost` 报 `UNKNOWN_ROLE` | + +仍有两条保持拒绝,因为服务器确实没有:`SHOW VARIABLES` 与 `SHOW STATUS`—— +`SHOW` 之后的语法里根本没有这两个关键字;配置在 `system.settings` / +`system.server_settings`,运行时计数器在 `system.metrics`、`system.events`、 +`system.asynchronous_metrics`,都由 `ClickHouseStatusIntrospector` 读取。 +`SHOW TABLE STATUS`、`SHOW TRIGGERS`、`SHOW CREATE TRIGGER`、`SHOW WARNINGS`、 +`SHOW ERRORS`、`SHOW CHARACTER SET`、`SHOW COLLATION`、`SHOW PLUGINS` 同理。 + +证明这些"不存在"的依据是服务器自己的语法。26.7.3.19 上 `SHOW` 接受的关键字全集是 +`CREATE`、`FULL`、`DATABASES`、`CLUSTERS`、`MERGES`、`FILESYSTEM CACHES`、 +`CLUSTER`、`CHANGED`、`SETTINGS`、`TEMPORARY`、`TABLES`、`DICTIONARIES`、 +`EXTENDED`、`COLUMNS`、`FIELDS`、`ENGINES`、`FUNCTIONS`、`INDEX`、`INDEXES`、 +`INDICES`、`KEYS`、`SETTING`、`DATABASE`、`VIEW`、`PROCESSLIST`、`ACCESS`、 +`USERS`、`ROLES`、`PROFILES`、`POLICIES`、`QUOTAS`、`MASKING POLICIES`、 +`CURRENT ROLES`、`ENABLED ROLES`、`CURRENT QUOTA`、`QUOTA`、`GRANTS`、 +`PRIVILEGES`——没有 `VARIABLES`、没有 `STATUS`、没有 `TRIGGERS`、没有 +`PLUGINS`。另见 。 + +### FOREIGN KEY 会被接受但不生效 + +`FOREIGN KEY` 值得单独说明,因为服务器**并非**简单拒绝它。四种写法中有三种直接 +被拒,第四种——也是通用 SQL 生成器最可能产出的那种——**被接受、静默丢弃、 +完全没有效果**。以下均在 26.7.3.19 上实测: + +| 写法 | 结果 | +|------|------| +| `CREATE TABLE c (id UInt32, pid UInt32, FOREIGN KEY (pid) REFERENCES p(id)) ENGINE = MergeTree ORDER BY id` | **成功。**`SHOW CREATE TABLE c` 里没有外键,`system.tables.create_table_query` 不含 `FOREIGN` 字符串,孤儿行可以写入并正常读回 | +| `CREATE TABLE c (id UInt32, CONSTRAINT fk FOREIGN KEY (id) REFERENCES p(id)) ...` | `Code: 62. Syntax error: failed at position 57 (FOREIGN) ... Expected one of: CHECK, ASSUME. (SYNTAX_ERROR)` | +| `CREATE TABLE c (id UInt32 REFERENCES p(id)) ...`(列内联引用)| `Code: 62. Syntax error: failed at position 43 (REFERENCES) ...` | +| `CREATE TABLE c (id UInt32) ENGINE = MergeTree ORDER BY id FOREIGN KEY (id) REFERENCES p(id)`(写在存储子句之后)| `Code: 62. Syntax error: failed at position 75 (FOREIGN) ...` | + +删掉被引用的父表同样成功,孤儿行依旧可读——所以这个子句也不是延迟检查,它是被 +彻底忽略了。 + +对本后端有两个后果: + +* **生成它会是三种结果里最糟的一种。**它不会报错,于是缺陷会在日后以"引用完整性 + 缺失"的形式出现在生产数据里,而服务器元数据中没有任何线索可以解释。这正是本后端 + [设计原则](../introduction/capability_boundaries.md)要避免的"静默模拟",也是 + `format_table_constraint` 对 `FOREIGN KEY` 抛异常、而不是渲染那个恰好能解析的 + 子句的原因。 +* **"DDL 不生成"与"服务器会拒绝它"不是同一个断言。**若读者把后者当作前提,就会 + 自己去 `CREATE TABLE` 里粘一个 `FOREIGN KEY` 试一下,然后对未命名的那种写法得出 + "服务器支持外键"的结论。 + +作为对照,ClickHouse *真正*支持的约束形式是 `CHECK` 与 `ASSUME`(均见 +), +且 `CHECK` 会被强制执行——越界的 `UInt32` 被按补码回绕存储而未被拒绝,仅仅是因为 +比较是在存储值上做的。而 `UNIQUE (col)` 则被直接拒绝:`Code: 62. Syntax error: +failed at position 50 ((): (col)) ... Expected one of: NULL, NOT, DEFAULT, +MATERIALIZED, EPHEMERAL, ALIAS, AUTO_INCREMENT, TTL, PRIMARY KEY, data type, +identifier. (SYNTAX_ERROR)`。 + +### CHECK TABLE 可用 + +`CHECK TABLE` 是 ClickHouse 的真实语句,本后端现在会生成它;它不在上面的快速失败 +清单里。它按表引擎分派,因此校验内容取决于引擎。 + +## 分区 + +ClickHouse **确实**支持分区,但方式与 MySQL 不同:在 `MergeTree` 表上以 +`PARTITION BY ` 声明,没有策略关键字,没有 `VALUES LESS THAN` / +`VALUES IN` / `MAXVALUE` 边界,也没有子分区。该子句通过 `storage_options` 声明; +维护按 partition id 进行(`DROP` / `DETACH` / `ATTACH PARTITION ID`); +自省走 `system.parts`。下表中的 MySQL 语句在 ClickHouse 没有对应写法, +因此是**彻底移除**而不是逐条关掉开关——证明其缺失的清单见 +。 + +| 功能 | 行为 | 替代 | +|------|------|------| +| 声明式 `PARTITION BY RANGE/LIST/HASH/KEY` + `PARTITION ... VALUES ...` | raise | `storage_options` 中的 `PARTITION BY ` | +| `ADD PARTITION` | raise | 分区在有数据落入时自然产生 | +| `TRUNCATE PARTITION` | raise | `ALTER TABLE ... DELETE IN PARTITION ... WHERE ...`,或 `TRUNCATE TABLE` | +| `REORGANIZE` / `EXCHANGE` / `COALESCE PARTITION`、`REMOVE PARTITIONING` | raise | `REPLACE PARTITION` / `MOVE PARTITION TO TABLE` | +| `ANALYZE`/`CHECK`/`OPTIMIZE`/`REBUILD`/`REPAIR PARTITION` | raise | `SYSTEM RESTART`/`SYSTEM RELOAD`、`OPTIMIZE TABLE ... PARTITION` | +| `SUBPARTITION BY` | raise | 元组分区键(`PARTITION BY (a, b)`)| +| `MAXVALUE` 分区边界 | raise | 分区键取值本身 | +| `information_schema.PARTITIONS` | raise | `system.parts` | ## 管理命令(MySQL admin set) -`FLUSH`/`RESET`/`CACHE INDEX`/`LOAD INDEX INTO CACHE`/`INSTALL`/`UNINSTALL COMPONENT`/`PLUGIN`/`CLONE`/`RESTART`/`BINLOG`/`HANDLER`/`DO`/`KILL`/`SHUTDOWN`/`HELP`/`CREATE USER`/`DROP USER`/`GRANT`/`REVOKE`——全部 raise。改用 ClickHouse 的 `SYSTEM` 命令族(`SYSTEM RELOAD`/`SYSTEM KILL`/`SYSTEM FLUSH`/...)。 +`FLUSH`/`RESET`/`CACHE INDEX`/`LOAD INDEX INTO CACHE`/`INSTALL`/`UNINSTALL COMPONENT`/`PLUGIN`/`CLONE`/`RESTART`/`BINLOG`/`HANDLER`/`DO`/`KILL`/`SHUTDOWN`/`HELP`——全部 raise。改用 ClickHouse 的 `SYSTEM` 命令族(`SYSTEM RELOAD`/`SYSTEM KILL`/`SYSTEM FLUSH`/...)。`KILL` 也不是 MySQL 那种单条语句:ClickHouse 的是 `KILL QUERY|MUTATION|PART_MOVE_TO_SHARD|TRANSACTION ...`,`KILL 1` 会得到 "Expected one of: QUERY, MUTATION, PART_MOVE_TO_SHARD, TRANSACTION"。 + +这份清单此前还列了 `CREATE USER`/`DROP USER`/`GRANT`/`REVOKE`,那几条并非 MySQL 专属: +`CREATE USER`、`ALTER USER`、`DROP USER`、`GRANT`、`REVOKE` 都是 ClickHouse 自己的 +访问控制语句,在 26.7.3.19 上全部执行成功——本后端只是没有可生成它们的核心 API, +因此不会把 `CREATE USER` 路由到服务端。从 SQL 控制台做访问管理并没有被禁止, +被禁止的只是本后端表达式层没有对应写法。 ## 异步 | 功能 | 行为 | 替代 | |------|------|------| -| 异步后端 | `AsyncClickHouseBackend` 实例化即抛 `NotImplementedError` | 同步后端 + 进程外并发 | - -`clickhouse-connect` 是纯同步库,本后端不提供 async 实现。 +| `AsyncClickHouseBackend` | 实例化即抛 `NotImplementedError` | 同步的 `ClickHouseBackend` + 进程外并发 | + +窄口径的说法是准确的:**没有任何 async 后端类可以建立连接。** +`AsyncClickHouseBackend(connection_config=...)` 在 `__init__` 里抛 +`NotImplementedError`,因为 `clickhouse-connect` 是纯同步驱动; +`AsyncClickHouseTransactionManager` 同样如此。 + +但"本包不提供 async 实现"是不对的。本包提供了两个完整的 async 自省器—— +`AsyncClickHouseStatusIntrospector` 与 `AsyncShowIntrospector`——它们各自实现了 +core 的 async status / SHOW 契约中的每个方法,都是真正 `await` 的查询, +没有一个方法抛 `NotImplementedError`。任何能自行提供 async 后端对象的调用方 +(自带的异步 ClickHouse 驱动,或一层包装)都能用到它们;只有随包提供的 +`AsyncClickHouseBackend` 拒绝实例化。异步测试 fixture 跳过也正是同一个原因: +它们拿不到一个可实例化的后端。 ## 如何探测 diff --git a/docs/zh_CN/ddl/skip_indexes.md b/docs/zh_CN/ddl/skip_indexes.md index 2ed6019..996126e 100644 --- a/docs/zh_CN/ddl/skip_indexes.md +++ b/docs/zh_CN/ddl/skip_indexes.md @@ -13,7 +13,7 @@ CREATE TABLE events ( ) ENGINE = MergeTree ORDER BY (ts) ``` -`INDEX ... USING type` 在本后端方言层支持。 +索引类型的关键字是 `TYPE`,不是 MySQL 的 `USING`:`INDEX i (s) TYPE minmax GRANULARITY 4` 会被接受,而 `INDEX i (s) USING minmax` 在 26.7.3.19 上是 `SYNTAX_ERROR`。 ## 常用 skip index 类型 @@ -23,22 +23,42 @@ CREATE TABLE events ( | `set(max_rows)` | 存储 granule 去重值集合 | 低基数 | | `bloom_filter` | 布隆过滤器判断可能存在 | 任意等值 | | `bloom_filter(bf)` | 带参数布隆 | 同上 | -| `tokenbf_v1` | token 分词布隆 | 文本 token 检索 | -| `ngrambf_v1(n, sz, b, f)` | n-gram 布隆 | 子串检索 | +| `text(tokenizer = ...)` | 分词后的倒排索引 | 全文检索 | +| `ngrambf_v1(n, sz, b, f)` | n-gram 布隆(自 26.2 起全文用途废弃) | 子串检索 | +| `tokenbf_v1(sz, f, seed)` | token 分词布隆(自 26.2 起全文用途废弃) | 单词检索 | + +服务端自己接受的清单就写在报错文本里:声明未知类型会得到 +``Unknown Index type 'fulltext'. Available index types: hypothesis, text, +vector_similarity, bloom_filter, sparse_grams, tokenbf_v1, ngrambf_v1, set, +minmax``。 ## 替代 FULLTEXT -ClickHouse **不支持** MySQL 的 `FULLTEXT` 索引与 `MATCH...AGAINST`(本后端 fail-fast)。用 `tokenbf_v1` + `hasToken` 做文本检索: +ClickHouse **不支持** MySQL 的 `FULLTEXT` 索引与 `MATCH...AGAINST`(本后端 fail-fast)。 +它的全文设施是 `text` 倒排索引,26.2 起 GA: ```sql CREATE TABLE docs ( body String, - INDEX idx_body body TYPE tokenbf_v1(3, 256, 2, 0) GRANULARITY 4 -) ENGINE = MergeTree ORDER BY id; + INDEX idx_body body TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 4 +) ENGINE = MergeTree ORDER BY tuple(); -SELECT * FROM docs WHERE hasToken(body, 'clickhouse'); +SELECT * FROM docs WHERE hasAllTokens(body, ['clickhouse', 'index']); +SELECT * FROM docs WHERE hasAnyTokens(body, ['index', 'inverted']); ``` +`text` 索引接受 `GRANULARITY` 但会忽略它——索引按整个 part 构建, +`SHOW CREATE TABLE` 里显示的是 `GRANULARITY 100000000`。 + +`tokenbf_v1` + `hasToken` 在 26.7.3.19 上仍然可用(`CREATE TABLE ... TYPE +tokenbf_v1(32768, 3, 0)` 被接受,`hasToken(body, 'beta')` 也能命中),但 +ClickHouse 自己的文档记录了 `tokenbf_v1` 与 `ngrambf_v1` 自 26.2 起在全文检索用途上 +已废弃、应由 `text` 取代,因此新表请优先用 `text`。参见 +。 + +`splitByNonAlpha` 区分大小写:需要忽略大小写时在索引上加 +`preprocessor = lower(body)`。 + ## 向量近似最近邻(替代 MySQL VECTOR index) 用 `vector_similarity` skip index(配合 `Array(Float32)` 列)做 ANN 检索,而非 MySQL 的 `VECTOR` 类型与 `DISTANCE_*` 函数。 diff --git a/docs/zh_CN/introduction/capability_boundaries.md b/docs/zh_CN/introduction/capability_boundaries.md index e45af01..4d1c146 100644 --- a/docs/zh_CN/introduction/capability_boundaries.md +++ b/docs/zh_CN/introduction/capability_boundaries.md @@ -8,7 +8,9 @@ - 静默忽略(如把 `transaction()` 变成空操作但对外声称已提交); - 退化成低效模拟(如用 `SELECT + DELETE + INSERT` 模拟 UPSERT); -- 生成 ClickHouse 会拒绝的 SQL(如 `FOREIGN KEY`)。 +- 生成 ClickHouse 会拒绝的 SQL(如 `UNIQUE (col)`),或生成它会接受却静默忽略的 + SQL(如未命名的 `FOREIGN KEY (...)`——见 + [FOREIGN KEY 会被接受但不生效](../capabilities/unsupported.md#foreign-key-会被接受但不生效))。 唯一例外是 `transaction()`:它降级为**空操作上下文管理器**,仅为让通用代码路径(核心库与 testsuite 的事务相关测试)能继续运行而不报错;但 **rollback 语义不存在**——mutations 是 per-part 原子的,无法跨语句回滚。 @@ -28,11 +30,13 @@ except UnsupportedFeatureError as e: ### ClickHouse 原生支持(一等公民) -- 列式类型:`Int*`/`UInt*`、`Float*`、`Decimal*`、`String`/`FixedString`、`Date`/`Date32`/`DateTime`/`DateTime64`、`Bool`、`UUID`、`IPv4`/`IPv6`、`Enum8`/`Enum16`、`Array`、`Map`、`Tuple`、`Nullable(T)`、`LowCardinality(T)`、`JSON` -- DDL:`ENGINE`、`ORDER BY` 排序键、`PARTITION BY`、`TTL`、skip indexes(`INDEX ... USING`) -- 查询:CTE(`WITH`)、窗口函数、`QUALIFY`、`FINAL`、`ARRAY JOIN`、集合操作(`UNION`/`INTERSECT`/`EXCEPT`,显式 `ALL`/`DISTINCT`)、`EXPLAIN` -- JSON:`JSONExtractString`/`JSONExtractRaw`/`JSON_VALUE` 等原生函数(**非** MySQL arrow 运算符 `->`/`->>`) -- 自省:`system.*` 表(settings/metrics/replicas/processes/...)、`SHOW` 命令 +- 列式类型:`Int*`/`UInt*`、`Float*`、`Decimal*`、`String`/`FixedString`、`Date`/`Date32`/`DateTime`/`DateTime64`、`Bool`、`UUID`、`IPv4`/`IPv6`、`Enum8`/`Enum16`、`Array`、`Map`、`Tuple`、`Nullable(T)`、`LowCardinality(T)`、`JSON`,以及 26.x 起的 `Geometry`/`Point` +- DDL:`ENGINE`、`ORDER BY` 排序键、`PARTITION BY`、`TTL`、skip indexes(`INDEX name (col) TYPE GRANULARITY n`——关键字是 `TYPE`,不是 MySQL 的 `USING`) +- 按 partition id 维护分区:`DROP` / `DETACH` / `ATTACH PARTITION ID`(ClickHouse 自己的子句清单见 ) +- 查询:CTE(`WITH`)、窗口函数、`QUALIFY`、`ARRAY JOIN`、集合操作(`UNION`/`INTERSECT`/`EXCEPT`,显式 `ALL`/`DISTINCT`)、`EXPLAIN`。`FINAL` 也是查询修饰符,但只对会合并重复行的引擎有效——普通 `MergeTree` 会返回 `Storage MergeTree doesn't support FINAL`,`ReplacingMergeTree` 则接受 +- JSON:`JSONExtractString`/`JSONExtractRaw`/`JSON_VALUE`(`JSON_VALUE` 只接受 `String` 列,传 `JSON` 类型会报 `JSONPath functions require first argument to be JSON of string`)等原生函数(**非** MySQL arrow 运算符 `->`/`->>`) +- 自省:`system.*` 表(settings/server_settings/metrics/events/asynchronous_metrics/replicas/processes/...)、`SHOW` 命令 +- SQL 层访问控制:`CREATE USER` / `ALTER USER` / `DROP USER` / `GRANT` / `REVOKE`(本后端表达式层没有可生成它们的写法,因此这里不会发出这些语句——但服务端有) - 客户端雪花 `Int64` id(替代 `AUTO_INCREMENT`) - 轻量级 UPDATE/DELETE(`UPDATE ... WHERE` / `DELETE ... WHERE`;需表设置 `enable_block_number_column`/`enable_block_offset_column`,详见 [变更](../capabilities/mutations.md)) @@ -41,23 +45,26 @@ except UnsupportedFeatureError as e: | 能力 | 行为 | 替代方案 | |------|------|---------| | ACID 跨语句事务 | `transaction()` 空操作;rollback 抛异常 | 按 per-part 原子变更设计,或用 `ReplacingMergeTree` | -| `FOREIGN KEY` / `UNIQUE` 约束 | DDL 不生成 | 由表引擎去重 | +| `FOREIGN KEY` / `UNIQUE` 约束 | DDL 不生成。`UNIQUE (col)` 在服务器上是 `SYNTAX_ERROR`;未命名的 `FOREIGN KEY (...) REFERENCES ...` 则**会被接受并静默丢弃**——见 [FOREIGN KEY 会被接受但不生效](../capabilities/unsupported.md#foreign-key-会被接受但不生效) | 由表引擎去重;引用完整性在应用层维护 | | 触发器、序列 | raise | — | | UPSERT / `ON CONFLICT` / `INSERT IGNORE` / `REPLACE INTO` | raise | `ReplacingMergeTree` + 显式 `INSERT` | | `FOR UPDATE` 行锁 | raise | OLAP 不做悲观锁 | -| `FULLTEXT` 索引 / `MATCH...AGAINST` | raise | skip indexes(`tokenbf_v1` + `hasToken`) | +| `FULLTEXT` 索引 / `MATCH...AGAINST` | raise | `text` 倒排索引(26.2 起 GA)+ `hasAllTokens`/`hasAnyTokens` | | `JSON_TABLE` | raise | `JSONExtract*` / `arrayJoin` | -| 空间类型(`GEOMETRY`/`POINT`/...)与 `ST_*` 函数 | raise | 存 WKT 为 `String`,用 ClickHouse 函数算 | -| MySQL 9.0 `VECTOR` 类型与 `STRING_TO_VECTOR`/`VECTOR_DIM`/`DISTANCE_*` | raise | `Array(Float32)` + `L2Distance`/`cosineDistance` | +| 空间类型(`GEOMETRY`/`POINT`/...)与 OGC `ST_*` 函数 | raise | ClickHouse 自己的 `Geometry`/`Point` 列类型与 7 个 WKB/MVT `ST_*` 编解码函数;距离用 `geoDistance` | +| MySQL 9.0 `VECTOR` 类型与 `STRING_TO_VECTOR`/`VECTOR_DIM`/`DISTANCE_*` | 不提供——该 API 已移除(ClickHouse 自己的向量类型是 `QBit`,由 `ClickHouseVectorType` 渲染) | `QBit(Float32, n)` 或 `Array(Float32)` + `L2Distance`/`cosineDistance` | | `SET` 类型 / `FIND_IN_SET` | raise | `Enum16` 或 `Array(String)` + `has()` | | 存储过程 / 存储函数 / `CALL` | raise | ClickHouse SQL UDF(`CREATE FUNCTION ... AS`)| | `LOAD DATA INFILE` / `LOAD XML` | raise | 格式解析器 / `input()` 表函数 | -| MySQL 管理命令(`FLUSH`/`RESET`/`KILL`/`INSTALL PLUGIN`/`CLONE`/`BINLOG`/`HANDLER`/`GRANT`/`CREATE USER`) | raise | ClickHouse `SYSTEM` 命令族 | +| MySQL 管理命令(`FLUSH`/`RESET`/`KILL`/`INSTALL PLUGIN`/`CLONE`/`BINLOG`/`HANDLER`) | raise | ClickHouse `SYSTEM` 命令族 | | `TABLE` / `VALUES` 表值构造(MySQL 8.0.19+) | raise | `SELECT ... UNION ALL SELECT ...` | -| 整表维护(`ANALYZE`/`CHECK`/`CHECKSUM`/`REPAIR TABLE`) | raise | `OPTIMIZE TABLE ... FINAL` 或 `SYSTEM` 命令 | -| MySQL optimizer hints(`/*+ SET_VAR */`) | raise | `SETTINGS` 子句 | +| 整表维护(`ANALYZE`/`CHECKSUM`/`REPAIR TABLE`) | raise(`CHECK TABLE` 是例外,它确实可用) | `OPTIMIZE TABLE ... FINAL` 或 `SYSTEM` 命令 | +| MySQL optimizer hints(`/*+ SET_VAR */`) | raise。这个拒绝并非多余:ClickHouse **会解析该注释但丢弃其内容**,26.7.3.19 上实测——`/*+ SET_VAR(max_block_size=7) */` 之后 `getSetting('max_block_size')` 仍是 `65409`;而 `SET_VAR(totally_bogus_zzz=1)` 这种根本不存在的设置名同样被接受,说明连设置名都不校验 | `SETTINGS` 子句 | | JSON Relational Duality Views | raise | `JSON` 类型 + `JSONExtract*` | -| 异步后端 | `AsyncClickHouseBackend` 占位类实例化即抛 `NotImplementedError` | 用同步后端 + 进程外并发 | +| ClickHouse 没有的 `SHOW` 语句(`SHOW VARIABLES`/`STATUS`/`TABLE STATUS`/`TRIGGERS`/`CREATE TRIGGER`/`WARNINGS`/`ERRORS`/`CHARACTER SET`/`COLLATION`/`PLUGINS`) | raise | 各自的 `system.*` 表,经 `ClickHouseStatusIntrospector` 读取。`SHOW` 之后根本没有 `VARIABLES`、`STATUS`、`TRIGGERS`、`PLUGINS` 这些关键字 | +| ClickHouse **确实**有的 `SHOW` 语句(`SHOW [FULL] COLUMNS`/`INDEX`/`PROCESSLIST`/`ENGINES`/`GRANTS`) | **不再拒绝——这几条可用**,经 `backend.introspector.show.*()` 读取。它们的列名是 ClickHouse 的,按 MySQL 的列名去解析只会拿到 `None` 而不是报错 | 若想直接查系统表就用对应的 `system.*` 表。见 [ClickHouse 实际具备的 `SHOW` 集合](../capabilities/unsupported.md#clickhouse-实际具备的-show-集合) | +| 声明式分区(`PARTITION BY RANGE/LIST/HASH/KEY` + `PARTITION ... VALUES ...`、子分区、`ADD`/`TRUNCATE`/`REORGANIZE`/`EXCHANGE`/`COALESCE`/`ANALYZE`/`CHECK`/`REBUILD`/`REPAIR PARTITION`) | raise——ClickHouse 没有这些子句,见 [分区](../capabilities/unsupported.md) | `storage_options` 中的 `PARTITION BY `;维护用 `DROP`/`DETACH`/`ATTACH PARTITION ID`;自省用 `system.parts` | +| 异步后端 | 没有可实例化的异步后端:`AsyncClickHouseBackend(...)` 直接抛 `NotImplementedError` | 用同步后端 + 进程外并发。本包内的两个异步*自省器*(`AsyncClickHouseStatusIntrospector`、`AsyncShowIntrospector`)是完整的、会真正 `await` 查询——只是没有随包后端可供它们运行 | 完整清单见 [不支持的功能](../capabilities/unsupported.md)。 diff --git a/requirements-dev.txt b/requirements-dev.txt index 76fcc7b..5873d2c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,4 +17,18 @@ pytest-cov # Web / HTTP for benchmarks and tests fastapi>=0.110 -httpx>=0.27 \ No newline at end of file +httpx>=0.27 + +# Python 3.13, including the free-threaded 3.13t build. The python_version +# marker distinguishes release lines, not GIL configuration, so these two lines +# pin both 3.13 builds. pydantic-core has no cp313t wheel after 2.41.4, and the +# newer sdists cannot be built there (PyO3 0.29 refuses the free-threaded build +# of CPython < 3.14); 2.12.2 is the last pydantic paired with 2.41.4. +# +# These mirror the same pins in rhosocial-activerecord's own dependencies, and +# they have to be restated here because this file is installed before the core +# package: resolving an unpinned `pydantic` here picks a pydantic-core whose only +# cp313t distribution is an sdist, and the build then fails on the free-threaded +# jobs before the core's pin is ever in scope. +pydantic[email]==2.12.2; python_version == '3.13' +pydantic-core==2.41.4; python_version == '3.13' diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/README.md b/src/rhosocial/activerecord/backend/impl/clickhouse/README.md index 9fa9f43..f771f24 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/README.md +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/README.md @@ -86,7 +86,7 @@ python -m rhosocial.activerecord.backend.impl.clickhouse \ ```json [ { - "VERSION()": "8.0.43" + "VERSION()": "26.7.3.19" } ] ``` \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/adapters.py b/src/rhosocial/activerecord/backend/impl/clickhouse/adapters.py index 4d3d198..5b95c0a 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/adapters.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/adapters.py @@ -303,9 +303,13 @@ def __init__(self, clickhouse_version: Optional[Tuple[int, int, int]] = None): """ Args: clickhouse_version: ClickHouse server version tuple (major, minor, patch). - If None, defaults to (8, 0, 0). + This adapter is not version-dependent — the canonical + ``'%Y-%m-%d %H:%M:%S.%f'`` text form it emits is accepted by + every maintained line — so ``None`` is recorded as ``None`` + rather than substituted with an invented version. ClickHouse's + numbering has never had an 8.0. """ - self._clickhouse_version = clickhouse_version or (8, 0, 0) + self._clickhouse_version = clickhouse_version @property def supported_types(self) -> Dict[Type, List[Any]]: @@ -522,179 +526,3 @@ def from_database( "and always fails. Use Array/Tuple columns instead." ) - -class ClickHouseVectorAdapter(SQLTypeAdapter): - """ - Adapts Python list of floats to ClickHouse VECTOR type and vice-versa. - - ClickHouse VECTOR type (9.0+) is used for AI/ML applications to store - multi-dimensional vectors. Supports up to 16,384 dimensions. - - Storage format: - - Binary format internally (optimized for similarity operations) - - Can be read/written as string representation '[1.0,2.0,3.0]' - - Supported distance functions: - - DISTANCE_EUCLIDEAN: Euclidean (L2) distance - - DISTANCE_COSINE: Cosine similarity distance - - DISTANCE_DOT: Dot product distance - """ - - # Maximum dimension supported by ClickHouse 9.0 - MAX_VECTOR_DIMENSION = 16384 - - def __init__(self, dimension: Optional[int] = None): - """ - Initialize VECTOR adapter. - - Args: - dimension: Optional expected vector dimension for validation. - If None, no dimension validation is performed. - """ - self._dimension = dimension - - @property - def supported_types(self) -> Dict[Type, List[Any]]: - # list[float] -> VECTOR (stored as binary or string) - return {list: [bytes, str]} - - def to_database(self, value: List[float], target_type: Type, options: Optional[Dict[str, Any]] = None) -> Any: - """ - Convert Python list of floats to ClickHouse VECTOR format. - - Args: - value: List of float values - target_type: Target database type (bytes or str) - options: Optional settings: - - 'dimension': Override expected dimension for validation - - Returns: - String representation '[v1,v2,...]' or binary format - - Raises: - ValueError: If dimension exceeds maximum or doesn't match expected - TypeError: If value contains non-float elements - """ - if value is None: - return None - - dimension = options.get("dimension", self._dimension) if options else self._dimension - - if len(value) > self.MAX_VECTOR_DIMENSION: - raise ValueError( - f"Vector dimension {len(value)} exceeds maximum supported dimension {self.MAX_VECTOR_DIMENSION}" - ) - - if dimension is not None and len(value) != dimension: - raise ValueError(f"Vector dimension {len(value)} doesn't match expected dimension {dimension}") - - # Validate all elements are floats or can be converted - for i, v in enumerate(value): - if not isinstance(v, (int, float)): - raise TypeError(f"Vector element at index {i} is not a number: {type(v).__name__}") - - # ClickHouse accepts string format '[1.0,2.0,3.0]' for VECTOR - # or use STRING_TO_VECTOR function - vector_str = "[" + ",".join(str(float(v)) for v in value) + "]" - - if target_type is bytes: - return vector_str.encode("utf-8") - return vector_str - - def _decode_vector_from_bytes(self, value: bytes) -> List[float]: - """ - Decode ClickHouse VECTOR from binary format. - - Args: - value: Binary data (either UTF-8 encoded string or packed floats) - - Returns: - List of float values - - Raises: - ValueError: If binary format is invalid - """ - # Try UTF-8 decode first (string format stored as bytes) - try: - return self._decode_vector_from_string(value.decode("utf-8")) - except UnicodeDecodeError: - pass - - # Binary format: packed IEEE 754 float32 values (little-endian) - import struct - - float_count = len(value) // 4 - if len(value) % 4 != 0: - raise ValueError( - f"Invalid VECTOR binary length: {len(value)} bytes (must be multiple of 4 for float32 values)" - ) from None - return list(struct.unpack(f"<{float_count}f", value)) - - def _decode_vector_from_string(self, value: str) -> List[float]: - """ - Decode ClickHouse VECTOR from string format. - - Args: - value: String representation like '[1.0,2.0,3.0]' - - Returns: - List of float values - - Raises: - ValueError: If string cannot be parsed - """ - # Remove brackets and split - value = value.strip() - if value.startswith("[") and value.endswith("]"): - value = value[1:-1] - - if not value: - return [] - - # Split by comma and convert to floats - try: - return [float(v.strip()) for v in value.split(",")] - except ValueError as e: - raise ValueError(f"Cannot parse VECTOR value: {value}") from e - - def from_database( - self, - value: Any, - target_type: Type, - options: Optional[Dict[str, Any]] = None, - **kwargs, - ) -> Optional[List[float]]: - """ - Convert ClickHouse VECTOR to Python list of floats. - - Args: - value: Database value (bytes, str, or already parsed list) - target_type: Target Python type (list) - options: Optional settings (currently unused) - - Returns: - List of float values - - Raises: - TypeError: If target_type is not list - ValueError: If value cannot be parsed - """ - if value is None: - return None - - if target_type is not list: - raise TypeError(f"ClickHouse VECTOR adapter only supports list target type, got {target_type.__name__}") - - # Already a list (some drivers might parse it) - if isinstance(value, list): - return [float(v) for v in value] - - # Binary format - if isinstance(value, bytes): - return self._decode_vector_from_bytes(value) - - # String format - if isinstance(value, str): - return self._decode_vector_from_string(value) - - raise TypeError(f"Cannot convert {type(value).__name__} to vector (list of floats)") diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/backend/backend.py b/src/rhosocial/activerecord/backend/impl/clickhouse/backend/backend.py index c1cf783..c265716 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/backend/backend.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/backend/backend.py @@ -436,8 +436,11 @@ def disconnect(self) -> None: conn.close() self.log(logging.INFO, "Disconnected from ClickHouse database") except (ClickHouseError, BrokenPipeError, OSError) as e: - # ClickHouse 5.6 may raise BrokenPipeError when closing a dead connection - # after KILL CONNECTION. We treat disconnect as always successful + # Closing a connection the server has already dropped (e.g. after + # KILL CONNECTION) writes to a socket the peer has closed, and + # the OS reports that as BrokenPipeError/OSError. That comes from + # the socket layer, not from a server release, so it is treated + # as expected at any version: disconnect is always successful # since the reference is already cleared. self.log(logging.WARNING, f"Error during disconnection (ignored): {str(e)}") diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/cli/info.py b/src/rhosocial/activerecord/backend/impl/clickhouse/cli/info.py index 58aa56a..eaf23e6 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/cli/info.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/cli/info.py @@ -212,12 +212,17 @@ def handle(args): # Create default dialect if not connected if dialect is None: - # Parse version from command line or use default + # Parse version from command line, else fall back to the oldest maintained + # line. 25.8 is a real ClickHouse LTS release (see + # docs/en_US/introduction/supported_versions.md) and is the safest floor to + # simulate: capability gates for anything newer stay off, so the output + # shows what an older server can do rather than what the newest can. The + # previous default was (8, 0, 0), a release ClickHouse has never had. actual_version = args.version if actual_version: version = parse_version(actual_version) else: - version = (8, 0, 0) # Default version + version = (25, 8, 0) from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect dialect = ClickHouseDialect(version=version) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/cli/status.py b/src/rhosocial/activerecord/backend/impl/clickhouse/cli/status.py index 68484f8..0c1c261 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/cli/status.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/cli/status.py @@ -175,7 +175,7 @@ def _handle_status_sync(args, backend: ClickHouseBackend, provider): data = _serialize_for_output(config_items) provider.display_results(data, title="Configuration") elif status_type == "performance": - perf_items = status_introspector.list_configuration(StatusCategory.PERFORMANCE) + perf_items = status_introspector.list_performance_metrics(StatusCategory.PERFORMANCE) data = _serialize_for_output(perf_items) provider.display_results(data, title="Performance") elif status_type == "connections": @@ -235,7 +235,7 @@ async def _handle_status_async(args, backend: AsyncClickHouseBackend, provider): data = _serialize_for_output(config_items) provider.display_results(data, title="Configuration") elif status_type == "performance": - perf_items = await status_introspector.list_configuration(StatusCategory.PERFORMANCE) + perf_items = await status_introspector.list_performance_metrics(StatusCategory.PERFORMANCE) data = _serialize_for_output(perf_items) provider.display_results(data, title="Performance") elif status_type == "connections": @@ -334,8 +334,10 @@ def _display_status_rich(status, verbose: int = 0): console.print(config_table) console.print() - # Performance section - perf_items = [item for item in status.configuration if item.category == StatusCategory.PERFORMANCE] + # Performance section. ServerOverview keeps performance metrics in + # `performance`; reading `configuration` here would print the setting list a + # second time under the wrong heading. + perf_items = [item for item in status.performance if item.category == StatusCategory.PERFORMANCE] if perf_items: console.print("[bold green]Performance[/bold green]") perf_table = Table(show_header=True, header_style="bold") diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/dialect.py b/src/rhosocial/activerecord/backend/impl/clickhouse/dialect.py index da2e44c..eb48d83 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/dialect.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/dialect.py @@ -111,13 +111,24 @@ DomainMixin, TransactionControlMixin, SetOperationMixin, + # ``LPAD``/``RPAD``/``REPEAT`` are nodes with default formatters, and + # ClickHouse spells all three natively -- the aliases ``lpad``/``rpad`` of + # ``leftPad``/``rightPad`` (21.8+) and ``repeat`` (20.1+) -- so the shared + # defaults are the answer for those three and no override is needed. + # ``TRIM`` is the one that differs; see ClickHouseTrimMixin. + LpadMixin, + RepeatMixin, + RpadMixin, + TrimMixin, + # Column types: the generic half, which ClickHouseColumnTypeMixin below + # overrides. + ColumnTypeMixin, ) from .protocols import ( ClickHouseTriggerSupport, ClickHouseTableSupport, ClickHouseJSONFunctionSupport, ClickHouseSpatialSupport, - ClickHouseVectorSupport, ClickHouseDMLOperationSupport, ClickHouseFullTextSearchSupport, ClickHouseLockingSupport, @@ -131,6 +142,7 @@ ClickHouseRoutineSupport, ClickHouseLoadXMLSupport, ClickHouseAdminCommandSupport, + ClickHouseTypeSupport, ) from .mixins import ( ClickHouseTransactionMixin, @@ -140,7 +152,6 @@ ClickHouseTableMixin, ClickHouseJSONFunctionMixin, ClickHouseSpatialMixin, - ClickHouseVectorMixin, ClickHouseIntrospectionMixin, ClickHouseLockingMixin, ClickHouseModifyColumnMixin, @@ -180,6 +191,11 @@ ClickHouseConstraintMixin, ClickHouseDDLColumnMixin, ClickHouseFunctionMixin, + ClickHouseColumnTypeMixin, + # TRIM differs and is overridden by ClickHouseTrimMixin (above) for the + # reason documented there: the direction is the function name, and the + # no-characters ANSI form the default renders is a syntax error here. + ClickHouseTrimMixin, ) from .reserved_words import CLICKHOUSE_RESERVED_WORDS from .show.dialect import ClickHouseShowDialectMixin @@ -287,13 +303,15 @@ class ClickHouseDialect( ClickHouseTransactionMixin, ClickHouseTableMixin, ClickHouseRenameTableMixin, + # Before TableMixin: this answers False where the core answers True, and it + # sat in the backend block below, so DROP TABLE was rendered with CASCADE + # and RESTRICT that ClickHouse does not accept. + ClickHouseConstraintMixin, TableMixin, ClickHouseTruncateMixin, TruncateMixin, - ClickHouseConstraintMixin, ConstraintMixin, ClickHouseSpatialMixin, - ClickHouseVectorMixin, ClickHouseIntrospectionMixin, ClickHouseShowDialectMixin, ClickHouseModifyColumnMixin, @@ -324,6 +342,23 @@ class ClickHouseDialect( ClickHouseSetOperationMixin, SetOperationMixin, ClickHouseFunctionMixin, + # Column types. The ClickHouse half first: it overrides the eighteen-entry + # table, and C3 gives the earlier name priority. Nothing else in this list + # can answer it, so the pair may move as a unit without disturbing the order + # around it -- which is the reason for not standing it next to + # ClickHouseTypeSupportMixin higher up, where a mistake would reorder the + # DDL formatters too. + ClickHouseColumnTypeMixin, + ColumnTypeMixin, + # TRIM is the one of the four string nodes ClickHouse spells differently: + # the direction lives in the function name, so ClickHouseTrimMixin must + # outrank core's TrimMixin in the MRO. LPAD, RPAD and REPEAT are the + # shared spellings verbatim here. + ClickHouseTrimMixin, + LpadMixin, + RepeatMixin, + RpadMixin, + TrimMixin, # Protocols for type checking CollationSupport, CTESupport, @@ -359,7 +394,6 @@ class ClickHouseDialect( TransactionControlSupport, ClickHouseTriggerSupport, ClickHouseSpatialSupport, - ClickHouseVectorSupport, ClickHouseFullTextSearchSupport, ClickHouseModifyColumnSupport, ClickHouseJsonDualityViewSupport, @@ -373,6 +407,10 @@ class ClickHouseDialect( ClickHouseLoadXMLSupport, ClickHouseAdminCommandSupport, SQLFunctionSupport, + # Before the generic DataTypeSupport it extends, for the same reason + # ClickHouseTypeSupportMixin precedes the mixin it completes: the backend + # protocol refines the generic one and must be seen first in the MRO. + ClickHouseTypeSupport, DataTypeSupport, # Named objects. Each object protocol names the ``format__object`` # method its ``*NameMixin`` provides, and inherits NamespaceSupport, so they diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/cli_commands.sh b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/cli_commands.sh index edd6f77..13cf308 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/cli_commands.sh +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/cli_commands.sh @@ -177,7 +177,7 @@ run_named_expression() { echo "" echo "--- List with dialect version ---" - $PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.mysql.examples.named_expressions.order_expressions --dialect-version 8.0.0 2>/dev/null || echo "(No named expression examples found)" + $PYTHON_CMD named-expression --list rhosocial.activerecord.backend.impl.mysql.examples.named_expressions.order_expressions --dialect-version 26.7.1 2>/dev/null || echo "(No named expression examples found)" echo "" echo "--- Describe expression ---" diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/info.sh b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/info.sh index 1ed1c08..302abdf 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/info.sh +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/cli/bash/info.sh @@ -39,8 +39,8 @@ echo "--- JSON output ---" $PYTHON_CMD info -o json echo "" -echo "--- Info with specific MySQL version ---" -$PYTHON_CMD info --version 5.7.0 +echo "--- Info against a specific release line ---" +$PYTHON_CMD info --version 25.8.0 echo "" echo "--- Rich ASCII output ---" diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/named_expressions/order_expressions.py b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/named_expressions/order_expressions.py index a550e7e..2d39aa6 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/named_expressions/order_expressions.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/named_expressions/order_expressions.py @@ -26,6 +26,9 @@ Literal, QueryExpression, ) +from rhosocial.activerecord.backend.expression.predicates import ( # noqa: E402 + ComparisonPredicate, +) def get_order(dialect, order_id: int): @@ -34,7 +37,7 @@ def get_order(dialect, order_id: int): dialect, select=[Column(dialect, "id"), Column(dialect, "status"), Column(dialect, "user_id")], from_=NamedRelationRef(dialect, Table(dialect, "orders")), - where=Column(dialect, "id") == Literal(dialect, order_id), + where=ComparisonPredicate(dialect, "=", Column(dialect, "id"), Literal(dialect, order_id)), ) @@ -44,7 +47,9 @@ def check_inventory(dialect, order_id: int): dialect, select=[Column(dialect, "available")], from_=NamedRelationRef(dialect, Table(dialect, "inventory")), - where=Column(dialect, "order_id") == Literal(dialect, order_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "order_id"), Literal(dialect, order_id) + ), ) @@ -54,7 +59,9 @@ def reserve_inventory(dialect, order_id: int): dialect, select=[Column(dialect, "id"), Column(dialect, "available")], from_=NamedRelationRef(dialect, Table(dialect, "inventory")), - where=Column(dialect, "order_id") == Literal(dialect, order_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "order_id"), Literal(dialect, order_id) + ), ) @@ -64,7 +71,9 @@ def send_notification(dialect, user_id: int, type: str): dialect, select=[Column(dialect, "id")], from_=NamedRelationRef(dialect, Table(dialect, "notifications")), - where=Column(dialect, "user_id") == Literal(dialect, user_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "user_id"), Literal(dialect, user_id) + ), ) @@ -74,7 +83,9 @@ def process_payment(dialect, order_id: int, amount: float): dialect, select=[Column(dialect, "status"), Column(dialect, "transaction_id")], from_=NamedRelationRef(dialect, Table(dialect, "payments")), - where=Column(dialect, "order_id") == Literal(dialect, order_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "order_id"), Literal(dialect, order_id) + ), ) @@ -84,7 +95,9 @@ def release_inventory(dialect, order_id: int): dialect, select=[Column(dialect, "id")], from_=NamedRelationRef(dialect, Table(dialect, "inventory")), - where=Column(dialect, "order_id") == Literal(dialect, order_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "order_id"), Literal(dialect, order_id) + ), ) @@ -94,7 +107,9 @@ def create_order_record(dialect, order_id: int, user_id: int, amount: float): dialect, select=[Column(dialect, "id"), Column(dialect, "created_at")], from_=NamedRelationRef(dialect, Table(dialect, "order_records")), - where=Column(dialect, "order_id") == Literal(dialect, order_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "order_id"), Literal(dialect, order_id) + ), ) @@ -104,7 +119,9 @@ def confirm_inventory(dialect, order_id: int): dialect, select=[Column(dialect, "id")], from_=NamedRelationRef(dialect, Table(dialect, "inventory")), - where=Column(dialect, "order_id") == Literal(dialect, order_id), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "order_id"), Literal(dialect, order_id) + ), ) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/cte.py b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/cte.py index 9539696..ebc03d8 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/cte.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/cte.py @@ -1,5 +1,23 @@ """ -Common Table Expressions (CTE) - ClickHouse 8.0+. +Common Table Expressions (CTE) in ClickHouse. + +ClickHouse supports ``WITH AS [MATERIALIZED] ``, +and both plain and ``WITH RECURSIVE`` forms were measured working on 26.7.3.19 +(``WITH cte AS (SELECT 1 AS x) SELECT * FROM cte`` → one row; +``WITH RECURSIVE t AS (SELECT 1 AS n UNION ALL SELECT n+1 FROM t WHERE n<3) +SELECT sum(n) FROM t`` → 6). + +There is no version floor to state for the plain form — the ``WITH`` reference +page +(``https://clickhouse.com/docs/reference/statements/select/with``) documents it +with no version qualifier. The one version that matters here is for +**recursive** CTEs, and it comes from that same page: they rely on the query +analyzer, "introduced in version **24.3**, which is the default since that +version and mandatory since **26.9**"; on an older instance with the analyzer +disabled a recursive CTE raises ``(UNKNOWN_TABLE)`` or +``(UNSUPPORTED_METHOD)``. The ``MATERIALIZED`` keyword is a separate, +experimental feature that additionally needs the ``enable_materialized_cte`` +setting. This example demonstrates: 1. Basic CTE with WITH clause @@ -219,7 +237,9 @@ # SECTION: Summary # ============================================================ # Key points: -# 1. CTE requires ClickHouse 8.0+ +# 1. Plain `WITH name AS (subquery)` has no ClickHouse version floor; the +# recursive form needs the query analyzer (default since 24.3, mandatory +# since 26.9) — see https://clickhouse.com/docs/reference/statements/select/with # 2. Use CTEExpression to define CTEs # 3. Use WithQueryExpression to combine CTEs with a main query # 4. Set recursive=True for recursive CTEs (hierarchical queries) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/window.py b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/window.py index 746e125..81f4774 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/window.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/examples/query/window.py @@ -90,7 +90,7 @@ WindowSpecification, OrderByClause, ) -from rhosocial.activerecord.backend.expression.advanced_functions import WindowFunctionCall # noqa: E402 +from rhosocial.activerecord.backend.expression.core import FunctionCall # noqa: E402 query = QueryExpression( dialect=dialect, @@ -98,7 +98,7 @@ Column(dialect, "salesperson"), Column(dialect, "region"), Column(dialect, "amount"), - WindowFunctionCall( + FunctionCall( dialect, "ROW_NUMBER", window_spec=WindowSpecification( @@ -108,10 +108,10 @@ ), alias="row_num", ), - WindowFunctionCall( + FunctionCall( dialect, "SUM", - args=[Column(dialect, "amount")], + Column(dialect, "amount"), window_spec=WindowSpecification( dialect, partition_by=[Column(dialect, "region")], diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/__init__.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/__init__.py index 78ea8fe..3fc1986 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/__init__.py @@ -7,10 +7,12 @@ - ``json`` — ClickHouse ``JSONExtract*`` / ``JSONObject`` / ``JSONArray`` function expressions (ClickHouse JSON is accessed via functions, not MySQL arrow operators). -- ``partition`` — MySQL declarative partitioning expression classes, kept as - fail-fast stubs (ClickHouse uses ``PARTITION BY `` - inside ``CREATE TABLE``, handled by the table-engine layer, - not MySQL ``PARTITION ... VALUES`` syntax). +- ``partition`` — ClickHouse's own partition maintenance, addressed by + partition id (``DROP`` / ``DETACH`` / ``ATTACH PARTITION ID``). + ClickHouse has no declarative partitioning: it partitions a + MergeTree table with ``PARTITION BY `` inside + ``CREATE TABLE``, rendered by the table-engine layer, and a + partition appears when a row lands in it. - ``rename_table`` — ClickHouse ``RENAME TABLE``. - ``types`` — ClickHouse-native ``DataType`` subclasses for DDL. @@ -52,34 +54,9 @@ ClickHouseJSONSearchExpression, ) from .partition import ( - ClickHousePartitionStrategy, - ClickHousePartitionClause, - ClickHousePartitionMaxValue, - ClickHousePartitionValue, - ClickHousePartitionDefinition, - ClickHousePartitionByRange, - ClickHousePartitionByRangeColumns, - ClickHousePartitionByList, - ClickHousePartitionByListColumns, - ClickHousePartitionByHash, - ClickHousePartitionByKey, - ClickHouseAddPartitionExpression, ClickHouseDropPartitionExpression, - ClickHouseTruncatePartitionExpression, - ClickHouseReorganizePartitionExpression, - ClickHouseExchangePartitionExpression, - ClickHouseRemovePartitioningExpression, - ClickHouseCoalescePartitionExpression, - ClickHouseAnalyzePartitionExpression, - ClickHouseCheckPartitionExpression, - ClickHouseOptimizePartitionExpression, - ClickHouseRebuildPartitionExpression, - ClickHouseRepairPartitionExpression, - ClickHouseGetPartitionsExpression, - ClickHousePartitionNameListExpression, - ClickHouseSubpartitionStrategy, - ClickHouseSubpartitionDefinition, - ClickHouseSubpartitionClause, + ClickHouseDetachPartitionExpression, + ClickHouseAttachPartitionExpression, ) from .rename_table import ClickHouseRenameTableExpression from .spatial import ( @@ -93,16 +70,6 @@ ClickHouseSTWithinExpression, ClickHouseSpatialLiteralExpression, ) -from .vector import ( - ClickHouseCreateVectorIndexExpression, - ClickHouseDistanceCosineExpression, - ClickHouseDistanceDotExpression, - ClickHouseDistanceEuclideanExpression, - ClickHouseStringToVectorExpression, - ClickHouseVectorDimExpression, - ClickHouseVectorLiteralExpression, - ClickHouseVectorToStringExpression, -) # DataType subclasses for DDL from .types import ( @@ -173,44 +140,10 @@ "ClickHouseSTGeomFromWKBExpression", "ClickHouseSTWithinExpression", "ClickHouseSpatialLiteralExpression", - # Vector expressions - "ClickHouseCreateVectorIndexExpression", - "ClickHouseDistanceCosineExpression", - "ClickHouseDistanceDotExpression", - "ClickHouseDistanceEuclideanExpression", - "ClickHouseStringToVectorExpression", - "ClickHouseVectorDimExpression", - "ClickHouseVectorLiteralExpression", - "ClickHouseVectorToStringExpression", # Partition - "ClickHousePartitionStrategy", - "ClickHousePartitionClause", - "ClickHousePartitionMaxValue", - "ClickHousePartitionValue", - "ClickHousePartitionDefinition", - "ClickHousePartitionByRange", - "ClickHousePartitionByRangeColumns", - "ClickHousePartitionByList", - "ClickHousePartitionByListColumns", - "ClickHousePartitionByHash", - "ClickHousePartitionByKey", - "ClickHouseAddPartitionExpression", "ClickHouseDropPartitionExpression", - "ClickHouseTruncatePartitionExpression", - "ClickHouseReorganizePartitionExpression", - "ClickHouseExchangePartitionExpression", - "ClickHouseRemovePartitioningExpression", - "ClickHouseCoalescePartitionExpression", - "ClickHouseAnalyzePartitionExpression", - "ClickHouseCheckPartitionExpression", - "ClickHouseOptimizePartitionExpression", - "ClickHouseRebuildPartitionExpression", - "ClickHouseRepairPartitionExpression", - "ClickHouseGetPartitionsExpression", - "ClickHousePartitionNameListExpression", - "ClickHouseSubpartitionStrategy", - "ClickHouseSubpartitionDefinition", - "ClickHouseSubpartitionClause", + "ClickHouseDetachPartitionExpression", + "ClickHouseAttachPartitionExpression", # Rename table "ClickHouseRenameTableExpression", # DataType subclasses for DDL diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_duality_view.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_duality_view.py index 3b27551..d8a7b7d 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_duality_view.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_duality_view.py @@ -1,9 +1,37 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_duality_view.py """ -ClickHouse JSON Duality View expressions (ClickHouse 9.7+). - -Provides expression classes for CREATE/DROP JSON RELATIONAL DUALITY VIEW -and the JSON_DUALITY_OBJECT function used within them. +ClickHouse JSON Duality View expressions. + +Provides expression classes for ``CREATE JSON RELATIONAL DUALITY VIEW`` and +``DROP`` of it, plus the dataclasses describing a ``JSON_DUALITY_OBJECT`` call. + +**ClickHouse has no duality views at any version, and there is nothing to +name a version for.** Three measurements on 26.7.3.19, each an inventory rather +than an absence of evidence: + +* ``CREATE JSON RELATIONAL DUALITY VIEW v AS SELECT 1`` — + ``Code: 62. DB::Exception: Syntax error: failed at position 8 (JSON): JSON + RELATIONAL DUALITY VIEW v AS SELECT 1. Expected one of: OR REPLACE, + TEMPORARY, TABLE, DATABASE, sql security, DEFINER, SQL SECURITY, MATERIALIZED, + VIEW, DICTIONARY, WINDOW, USER, ROLE, QUOTA, POLICY, ROW POLICY, MASKING + POLICY, SETTINGS PROFILE, PROFILE, FUNCTION, WORKLOAD, RESOURCE, NAMED + COLLECTION, UNIQUE, INDEX, HYPOTHETICAL. (SYNTAX_ERROR)`` — that list is the + server enumerating every object ``CREATE`` can make, and a duality view is not + on it. +* ``SELECT JSON_DUALITY_OBJECT(1)`` — ``Code: 46. DB::Exception: Function with + name `JSON_DUALITY_OBJECT` does not exist. (UNKNOWN_FUNCTION)``, and + ``SELECT count() FROM system.functions WHERE name ILIKE '%DUALITY%'`` is 0. +* ``https://clickhouse.com/docs/reference/statements/create/table`` documents + exactly two table-constraint forms — ``CONSTRAINT name CHECK expr`` and + ``CONSTRAINT name ASSUME expr`` — and no duality view. + +So the switch stays fail-fast rather than becoming a version gate: +``supports_json_duality_view()`` is ``False`` and +``format_create_json_duality_view_statement`` raises +``UnsupportedFeatureError``. What the server does have for reacting to a row is +``CREATE FUNCTION ... AS`` (a SQL UDF) plus a MATERIALIZED VIEW, and what it has +for relational JSON is the ``JSON`` column type with the ``JSONExtract*`` +family. """ from __future__ import annotations @@ -74,7 +102,13 @@ class DualityObjectSpec: class CreateJsonDualityViewExpression(BaseExpression): - """CREATE JSON RELATIONAL DUALITY VIEW expression (ClickHouse 9.7+). + """``CREATE JSON RELATIONAL DUALITY VIEW`` — a statement ClickHouse does not have. + + Kept as a fail-fast switch, not as a renderer: ``to_sql()`` raises + ``UnsupportedFeatureError`` through the dialect, naming + ``CREATE FUNCTION ... AS`` plus a MATERIALIZED VIEW as the ClickHouse way to + react to a row. There is no ClickHouse version at which this statement + starts parsing — see the module docstring for the three measurements. Attributes: view_name: Name of the duality view diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_table.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_table.py index 6b39840..405d435 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_table.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/json_table.py @@ -4,6 +4,29 @@ This module provides ClickHouseJSONTableExpression, JSONTableColumn, and NestedPath for ClickHouse's JSON_TABLE functionality. + +**ClickHouse has no ``JSON_TABLE`` at any version**, so there is nothing to name +a version for. MySQL's is the SQL/JSON ``JSON_TABLE(expr, path COLUMNS (...) +NESTED PATH ...)`` table expression; three inventories on 26.7.3.19 rule out +every ClickHouse spelling of it: + +* ``SELECT count() FROM system.functions WHERE name = 'JSON_TABLE'`` is ``0``. +* ``SELECT name FROM system.table_functions WHERE name ILIKE '%json%'`` returns + one row, ``fuzzJSON`` — there is no ``json_table`` table function either. (The + table-function slot is what ClickHouse's JSON support looks like: the ``JSON`` + column type plus the ``JSONExtract*`` function family.) +* ``SELECT JSON_TABLE('[]', '$[*]' COLUMNS (a Int32))`` — + ``Code: 62. DB::Exception: Syntax error: failed at position 33 (COLUMNS): + COLUMNS (a Int32)). Expected one of: token, Comma, ClosingRoundBracket, OR, + AND, IS NOT DISTINCT FROM, ... (SYNTAX_ERROR)``. The parser got as far as + treating ``JSON_TABLE`` as an ordinary function call with two arguments and + then only wanted an operator; it never saw a ``COLUMNS`` clause. This is the + signature of an unknown function name, not of a dialect difference. + +So ``supports_json_table()`` is ``False`` and ``format_json_table_expression`` +raises ``UnsupportedFeatureError``, naming ``JSONExtract*`` as the ClickHouse +route. The class below stays as that fail-fast switch rather than as a +renderer. """ from dataclasses import dataclass @@ -54,10 +77,12 @@ class NestedPath: class ClickHouseJSONTableExpression(BaseExpression): - """ClickHouse JSON_TABLE expression. + """MySQL's ``JSON_TABLE`` table expression — a statement ClickHouse does not have. - Generates JSON_TABLE function for converting JSON data to relational format. - Supported in ClickHouse 8.0.4+. + Kept as a fail-fast switch: ``to_sql()`` raises ``UnsupportedFeatureError`` + through the dialect, pointing at the ``JSONExtract*`` family and the ``JSON`` + column type. No ClickHouse version parses this statement; see the module + docstring for the three inventories that establish it. Attributes: json_doc: JSON document string or expression diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/match_against.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/match_against.py index a42c125..dfa70a6 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/match_against.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/match_against.py @@ -33,7 +33,15 @@ class ClickHouseMatchAgainstExpression( """ClickHouse MATCH...AGAINST expression. Generates MATCH(col1, col2, ...) AGAINST(search_string [IN mode]) syntax. - Supported in ClickHouse 5.6+ (with FULLTEXT index). + + This dialect never renders it: ``ClickHouseFullTextSearchMixin.format_match_against`` + raises ``UnsupportedFeatureError``. ClickHouse has no ``MATCH ... AGAINST`` + and no ``FULLTEXT`` index type — declaring one answers ``Unknown Index type + 'fulltext'`` and lists what the server does have (``hypothesis, text, + vector_similarity, bloom_filter, sparse_grams, tokenbf_v1, ngrambf_v1, set, + minmax``). The ClickHouse equivalent is a ``text`` inverted index queried + with ``hasAllTokens`` / ``hasAnyTokens``. No ClickHouse version enables this + syntax, because no version has the statement. Attributes: columns: Column names to search diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/optimizer_hint.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/optimizer_hint.py index b66689b..965a487 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/optimizer_hint.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/optimizer_hint.py @@ -1,9 +1,34 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/expression/optimizer_hint.py """ClickHouse optimizer hint expressions. -Supports per-statement optimizer hints using the /*+ ... */ syntax, -including SET_VAR hints for controlling optimizer switches like -the hypergraph optimizer (ClickHouse 9.7+). +Models per-statement optimizer hints in the ``/*+ ... */`` comment syntax, +including ``SET_VAR`` hints that name a setting and a value. + +**The comment parses on 26.7.3.19 and the contents are discarded**, so there is +no version floor and no setting to name a version for. Five measurements: + +* ``SELECT /*+ SET_VAR(max_threads=1) */ 1`` succeeds — the comment is + recognised, so "ClickHouse cannot parse ``/*+ ... */``" would be wrong. +* ``SELECT /*+ SET_VAR(totally_bogus_zzz=1) */ 1`` also succeeds, so the hint's + setting name is not validated at parse time. +* ``SELECT /*+ SET_VAR(max_block_size=7) */ getSetting('max_block_size')`` + returns ``65409``, the session default: the hint has **no effect**. (The same + query with an explicit ``SETTINGS max_block_size=7`` does change it.) +* ``SELECT count() FROM system.settings`` is 1715 on this server, and + ``WHERE name ILIKE '%optimizer%'`` matches **none** of them. +* ``SELECT 1 SETTINGS optimizer_switch='hypergraph_optimizer=on'`` — + ``Code: 115. DB::Exception: Setting optimizer_switch is neither a builtin + setting nor started with the prefix 'SQL_' registered for user-defined + settings. (UNKNOWN_SETTING)``. There is no ``optimizer_switch`` here to carry + a ``hypergraph_optimizer`` flag. + +So the expression class below is a fail-fast switch, not a renderer: +``supports_optimizer_hint()`` and ``supports_hypergraph_optimizer()`` are both +``False`` and ``format_optimizer_hint`` raises ``UnsupportedFeatureError``. The +ClickHouse way to set something per statement is the ``SETTINGS`` clause on the +statement itself (``https://clickhouse.com/docs/reference/statements/select``, +"Settings in SELECT Query"), which this backend renders for the settings it +actually models. """ from __future__ import annotations @@ -33,14 +58,20 @@ class SetVarHint: class ClickHouseOptimizerHintExpression(BaseExpression): - """Expression for ClickHouse optimizer hints (/*+ ... */ syntax). + """A MySQL-style ``/*+ ... */`` optimizer hint — accepted but inert in ClickHouse. - Usage: + Kept as a fail-fast switch: ``to_sql()`` raises ``UnsupportedFeatureError`` + through the dialect. The server parses the comment and then ignores it (see + the module docstring for the measurements, including the ``SET_VAR`` case + that leaves ``max_block_size`` at its default), so a hint this backend + rendered would be a comment that does nothing — which is worse than a + refusal, because it looks like it worked. + + Usage (of the switch, not of a renderer): hint = ClickHouseOptimizerHintExpression(dialect, [ SetVarHint("optimizer_switch", "hypergraph_optimizer=on") ]) - sql, params = hint.to_sql() - # => ("/*+ SET_VAR(optimizer_switch='hypergraph_optimizer=on') */", ()) + hint.to_sql() # raises UnsupportedFeatureError """ def __init__(self, dialect: "SQLDialectBase", hints: List[SetVarHint]): diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition.py index 9d883c9..65d3187 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition.py @@ -1,682 +1,201 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition.py +"""ClickHouse table-partition DDL expressions. + +ClickHouse partitioning is **not** MySQL's declarative partitioning. It is a +property of the ``MergeTree`` family of table engines, declared as an +arbitrary expression in ``CREATE TABLE``:: + + CREATE TABLE visits (VisitDate Date, Hour UInt8) + ENGINE = MergeTree + PARTITION BY toYYYYMM(VisitDate) -- any expression, or a tuple of them + ORDER BY Hour + +There is no partitioning *strategy* to choose, no ``PARTITION ... VALUES LESS +THAN`` / ``VALUES IN`` / ``MAXVALUE`` boundary, and no subpartitioning: a +partition is simply the set of rows whose partition-key expression evaluates +to the same value. A partition appears the first time a row lands in it; it is +never declared, so it cannot be "added" as an empty range either. + +What ``ALTER TABLE`` does offer is *maintenance* of an existing partition, and +it addresses it by **partition id** (the human-readable string in the +``partition_id`` column of ``system.parts``) or by the partition-key value +itself:: + + ALTER TABLE visits DROP PARTITION ID '202601'; + ALTER TABLE visits DETACH PARTITION ID '202601'; + ALTER TABLE visits ATTACH PARTITION ID '202601'; + +Only those three appear in this module; the rest of the ClickHouse inventory +is on the page cited below (``DROP PART``, ``DROP DETACHED PARTITION``, +``FORGET PARTITION``, ``ATTACH PARTITION FROM``, ``REPLACE PARTITION``, +``MOVE PARTITION TO TABLE``, ``FREEZE``/``UNFREEZE``, ``FETCH PARTITION``, +``MOVE PARTITION|PART ... TO DISK|VOLUME``, ``CLEAR COLUMN|INDEX IN PARTITION``, +``UPDATE``/``DELETE IN PARTITION``, ``REWRITE PARTS``). A statement that is not +on that page has no ClickHouse spelling at any version. That inventory is also +what proves the absence of the MySQL statements this backend used to stub +(``ADD PARTITION``, ``TRUNCATE PARTITION``, ``REORGANIZE PARTITION``, +``EXCHANGE PARTITION``, ``REMOVE PARTITIONING``, ``COALESCE PARTITION``, +``ANALYZE``/``CHECK``/``OPTIMIZE``/``REBUILD``/``REPAIR PARTITION``, +``SUBPARTITION BY``): none of them is in it, and asking the server for one +returns an expectation list naming every clause it *does* accept. + +Introspecting partitions reads ``system.parts`` (``partition``, +``partition_id``, ``name``, ``active``); there is no +``information_schema.PARTITIONS`` — verified on the scenario server, which +answers that table name with ``Unknown table expression identifier +'information_schema.PARTITIONS'``. + +Declaring ``PARTITION BY`` is not this module's job: it belongs to the storage +clauses of the table engine, rendered from the ``storage_options`` mapping by +``ClickHouseTableEngineMixin.format_table_engine_clauses``. + +References (both fetched and HTTP-resolving): +https://clickhouse.com/docs/engines/table-engines/mergetree-family/custom-partitioning-key +https://clickhouse.com/docs/sql-reference/statements/alter/partition """ -ClickHouse partition DDL expressions. - -WARNING: This module contains MySQL declarative partitioning expression classes -that are dead code for ClickHouse. ClickHouse uses ``PARTITION BY`` expression -in ``CREATE TABLE`` (handled by ``ClickHouseTableEngineMixin``), not MySQL -declarative partitioning (RANGE/LIST/HASH/KEY). All ``to_sql()`` methods raise -``UnsupportedFeatureError``. -""" - -from __future__ import annotations - -from collections.abc import Sequence as AbcSequence -from dataclasses import dataclass -from decimal import Decimal -from enum import Enum -from math import isfinite -from typing import Any, List, Optional, Sequence, TYPE_CHECKING - -from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError -from rhosocial.activerecord.backend.expression.bases import BaseExpression, SQLQueryAndParams -from rhosocial.activerecord.backend.expression.objects import Table -from rhosocial.activerecord.backend.expression.statements import ( - PartitionClause, - PartitionDefinition, - SubpartitionDefinition, -) - - -class ClickHousePartitionStrategy(Enum): - """MySQL declarative partitioning strategies (not supported by ClickHouse). - - ClickHouse does not support these strategies; partition keys are expressed - via the ``PARTITION BY`` expression in ``CREATE TABLE``. Retained for - interface compatibility; SQL generation raises ``UnsupportedFeatureError``. - """ - - RANGE = "RANGE" - RANGE_COLUMNS = "RANGE COLUMNS" - LIST = "LIST" - LIST_COLUMNS = "LIST COLUMNS" - HASH = "HASH" - LINEAR_HASH = "LINEAR HASH" - KEY = "KEY" - LINEAR_KEY = "LINEAR KEY" - - -class ClickHouseSubpartitionStrategy(Enum): - """MySQL subpartitioning strategies. - - ClickHouse does not support subpartitioning. Retained for interface - compatibility; SQL generation raises ``UnsupportedFeatureError``. - """ - - HASH = "HASH" - KEY = "KEY" - LINEAR_HASH = "LINEAR HASH" - LINEAR_KEY = "LINEAR KEY" - - -@dataclass -class ClickHouseSubpartitionDefinition(SubpartitionDefinition): - """Dead-code MySQL subpartition definition, derived from the generic base. - - ClickHouse does not support MySQL declarative subpartitioning. Retained - for interface compatibility only; ``to_sql()`` paths raise - ``UnsupportedFeatureError``. - """ - - -class ClickHouseSubpartitionClause(BaseExpression): - """MySQL ``SUBPARTITION BY {HASH|KEY}(...) SUBPARTITIONS N`` clause. - - This is MySQL declarative partitioning syntax; ClickHouse does not support - it. Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - - Raises: - TypeError: if strategy is not a ClickHouseSubpartitionStrategy. - ValueError: if count is provided but is not a positive integer. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - strategy: ClickHouseSubpartitionStrategy, - *, - expression: Optional[BaseExpression] = None, - count: Optional[int] = None, - definitions: Optional[Sequence[ClickHouseSubpartitionDefinition]] = None, - ): - super().__init__(dialect) - if not isinstance(strategy, ClickHouseSubpartitionStrategy): - raise TypeError( - "strategy must be a ClickHouseSubpartitionStrategy value, " - f"got {type(strategy).__name__}" - ) - if count is not None and (not isinstance(count, int) or count <= 0): - raise ValueError("count must be a positive integer when provided") - self.strategy = strategy - self.expression = expression - self.count = count - self.definitions = list(definitions) if definitions else None - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: ClickHouse has no SUBPARTITION BY.""" - raise UnsupportedFeatureError( - self.dialect.name, - "SUBPARTITION BY", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) +from typing import Optional, TYPE_CHECKING +from rhosocial.activerecord.backend.expression.bases import BaseExpression if TYPE_CHECKING: # pragma: no cover from ..dialect import ClickHouseDialect -class ClickHousePartitionMaxValue(BaseExpression): - """MySQL MAXVALUE partition boundary token (not supported by ClickHouse). +__all__ = [ + "ClickHouseDropPartitionExpression", + "ClickHouseDetachPartitionExpression", + "ClickHouseAttachPartitionExpression", +] - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - def __init__(self, dialect: "ClickHouseDialect"): - super().__init__(dialect) +class _ClickHousePartitionIdExpression(BaseExpression): + """Shared shape for the ``ALTER TABLE ... PARTITION ID`` clauses. - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: ClickHouse has no MAXVALUE token.""" - raise UnsupportedFeatureError( - self.dialect.name, - "MAXVALUE partition boundary", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) + ClickHouse addresses a partition by **id**, not by a name declared in the + table's DDL: the id is the string the server derives from the partition key + and reports in ``system.parts.partition_id`` (for a table partitioned by + ``toYYYYMM(created_at)`` that is ``'202601'``). MySQL names its partitions in + the DDL and addresses them by that name; ClickHouse has no such name to + address, and a name passed where an expression belongs is a parse error — + on the scenario server ``ALTER TABLE t DROP PARTITION p2026_01`` answers + ``Expected one of: token sequence, Dot, token``. What the server does accept + is listed in the "How to Set Partition Expression" section of the ALTER + reference: a value from ``system.parts.partition``, the keyword ``ALL``, a + ``tuple(...)`` of key values, or ``PARTITION ID ''``. This backend + always renders the last of those, because the id form is the one that works + whatever the partition key's type is (``DROP PARTITION 202601`` happens to + parse for a numeric key, ``DROP PARTITION '202601'`` for a string one, but + only ``PARTITION ID`` needs no per-key guessing). - -class ClickHousePartitionValue(BaseExpression): - """Literal value used in MySQL partition boundary definitions. - - MySQL declarative partitioning is not supported by ClickHouse. Retained - for interface compatibility; ``to_sql()`` raises ``UnsupportedFeatureError``. + Subclasses supply only the verb; the rendering is shared so the three + clauses cannot drift apart. """ - def __init__(self, dialect: "ClickHouseDialect", value: Any): - super().__init__(dialect) - if isinstance(value, float) and not isfinite(value): - raise ValueError("partition value float must be finite") - if not isinstance(value, (str, int, float, Decimal, type(None))): - from datetime import date, datetime - - if not isinstance(value, (date, datetime)): - raise TypeError( - "partition value must be str, int, float, Decimal, " - f"date, datetime, or None, got {type(value).__name__}" - ) - self.value = value - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: ClickHouse has no partition VALUES.""" - raise UnsupportedFeatureError( - self.dialect.name, - "partition VALUES boundary", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -@dataclass -class ClickHousePartitionDefinition(PartitionDefinition): - """Dead-code MySQL ``PARTITION ... VALUES ...`` definition. - - ClickHouse does not support MySQL declarative partitioning; boundaries are - expressed via the ``PARTITION BY`` expression in ``CREATE TABLE``. Retained - for interface compatibility only; the formatter raises - ``UnsupportedFeatureError``. Derives from the generic base declaration. - - Raises: - ValueError: if both ``less_than`` and ``in_values`` are provided, - or if neither is provided. - """ - - subpartition_definitions: Optional[Sequence["ClickHouseSubpartitionDefinition"]] = None - - def __post_init__(self) -> None: - super().__post_init__() - if self.less_than is None and self.in_values is None: - raise ValueError("partition definition requires less_than or in_values") - - -class ClickHousePartitionClause(PartitionClause): - """Base ClickHouse partition clause with ClickHouse-specific strategy enum. - - MySQL declarative partitioning (RANGE/LIST/HASH/KEY) is not supported by - ClickHouse; partitioning is expressed via ``PARTITION BY`` in ``CREATE - TABLE``. ``to_sql()`` raises ``UnsupportedFeatureError``. - """ - - strategy_type = ClickHousePartitionStrategy - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: ClickHouse has no declarative partitioning.""" - raise UnsupportedFeatureError( - self.dialect.name, - f"{self.method} declarative partitioning", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHousePartitionByRange(ClickHousePartitionClause): - """MySQL ``PARTITION BY RANGE`` expression (not supported by ClickHouse). - - Raises: - TypeError: if subpartition_by is not a ClickHouseSubpartitionClause. - """ + #: The ``ALTER TABLE`` clause verb, overridden by each concrete subclass. + verb: str = "" def __init__( self, dialect: "ClickHouseDialect", - keys: Sequence[BaseExpression], + table: str, + partition_id: str, *, - partitions: Optional[Sequence[ClickHousePartitionDefinition]] = None, - subpartition_by: Optional[ClickHouseSubpartitionClause] = None, + schema: Optional[str] = None, ): - super().__init__(dialect, ClickHousePartitionStrategy.RANGE, keys) - if subpartition_by is not None and not isinstance(subpartition_by, ClickHouseSubpartitionClause): - raise TypeError("subpartition_by must be a ClickHouseSubpartitionClause") - self.partitions = list(partitions or []) - self.subpartition_by = subpartition_by - - -class ClickHousePartitionByRangeColumns(ClickHousePartitionClause): - """MySQL ``PARTITION BY RANGE COLUMNS`` expression (not supported by ClickHouse). - - Raises: - TypeError: if subpartition_by is not a ClickHouseSubpartitionClause. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - keys: Sequence[BaseExpression], - *, - partitions: Optional[Sequence[ClickHousePartitionDefinition]] = None, - subpartition_by: Optional[ClickHouseSubpartitionClause] = None, - ): - super().__init__(dialect, ClickHousePartitionStrategy.RANGE_COLUMNS, keys) - if subpartition_by is not None and not isinstance(subpartition_by, ClickHouseSubpartitionClause): - raise TypeError("subpartition_by must be a ClickHouseSubpartitionClause") - self.partitions = list(partitions or []) - self.subpartition_by = subpartition_by - - -class ClickHousePartitionByList(ClickHousePartitionClause): - """MySQL ``PARTITION BY LIST`` expression (not supported by ClickHouse). - - Raises: - TypeError: if subpartition_by is not a ClickHouseSubpartitionClause. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - keys: Sequence[BaseExpression], - *, - partitions: Optional[Sequence[ClickHousePartitionDefinition]] = None, - subpartition_by: Optional[ClickHouseSubpartitionClause] = None, - ): - super().__init__(dialect, ClickHousePartitionStrategy.LIST, keys) - if subpartition_by is not None and not isinstance(subpartition_by, ClickHouseSubpartitionClause): - raise TypeError("subpartition_by must be a ClickHouseSubpartitionClause") - self.partitions = list(partitions or []) - self.subpartition_by = subpartition_by - - -class ClickHousePartitionByListColumns(ClickHousePartitionClause): - """MySQL ``PARTITION BY LIST COLUMNS`` expression (not supported by ClickHouse). - - Raises: - TypeError: if subpartition_by is not a ClickHouseSubpartitionClause. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - keys: Sequence[BaseExpression], - *, - partitions: Optional[Sequence[ClickHousePartitionDefinition]] = None, - subpartition_by: Optional[ClickHouseSubpartitionClause] = None, - ): - super().__init__(dialect, ClickHousePartitionStrategy.LIST_COLUMNS, keys) - if subpartition_by is not None and not isinstance(subpartition_by, ClickHouseSubpartitionClause): - raise TypeError("subpartition_by must be a ClickHouseSubpartitionClause") - self.partitions = list(partitions or []) - self.subpartition_by = subpartition_by - - -class ClickHousePartitionByHash(ClickHousePartitionClause): - """MySQL ``PARTITION BY HASH`` expression (not supported by ClickHouse).""" - - def __init__( - self, - dialect: "ClickHouseDialect", - keys: Sequence[BaseExpression], - *, - partitions_count: Optional[int] = None, - linear: bool = False, - ): - method = ClickHousePartitionStrategy.LINEAR_HASH if linear else ClickHousePartitionStrategy.HASH - super().__init__(dialect, method, keys) - self.partitions_count = partitions_count - self.linear = linear - - -class ClickHousePartitionByKey(ClickHousePartitionClause): - """MySQL ``PARTITION BY KEY`` expression (not supported by ClickHouse). - - ClickHouse uses ``PARTITION BY`` expression in ``CREATE TABLE``, not - MySQL declarative KEY partitioning. Retained for interface compatibility; - ``to_sql()`` raises ``UnsupportedFeatureError``. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - keys: Optional[Sequence[BaseExpression]] = None, - *, - partitions_count: Optional[int] = None, - linear: bool = False, - ): - method = ClickHousePartitionStrategy.LINEAR_KEY if linear else ClickHousePartitionStrategy.KEY - if keys: - super().__init__(dialect, method, keys) - else: - BaseExpression.__init__(self, dialect) - self.method = method.value - self.keys = list(keys) if keys else [] - self.partitions_count = partitions_count - self.linear = linear - - -class ClickHouseAddPartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... ADD PARTITION`` expression. - - MySQL declarative partition maintenance is not supported by ClickHouse; - retained for interface compatibility. ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partitions: List[ClickHousePartitionDefinition], - ): - super().__init__(dialect) - self.table = table - self.partitions = partitions - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: ClickHouse has no ADD PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "ADD PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseDropPartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... DROP PARTITION`` expression. - - ClickHouse removes partitions via ``ALTER TABLE ... DROP PARTITION`` with - partition-id syntax, not MySQL declarative partition names. Retained for - interface compatibility; ``to_sql()`` raises ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) - self.table = table - self.partitions = list(partitions) - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative DROP PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "DROP PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseTruncatePartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... TRUNCATE PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) - self.table = table - self.partitions = list(partitions) - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative TRUNCATE PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "TRUNCATE PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseReorganizePartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... REORGANIZE PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partition: str, - into: List[ClickHousePartitionDefinition], - ): - super().__init__(dialect) - self.table = table - self.partition = partition - self.into = into - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative REORGANIZE PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "REORGANIZE PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseExchangePartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... EXCHANGE PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partition: str, - exchange_table: Table, - *, - with_validation: bool = True, - ): - super().__init__(dialect) - self.table = table - self.partition = partition - self.exchange_table = exchange_table - self.with_validation = with_validation - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative EXCHANGE PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "EXCHANGE PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseRemovePartitioningExpression(BaseExpression): - """MySQL ``ALTER TABLE ... REMOVE PARTITIONING`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table): - super().__init__(dialect) - self.table = table - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative REMOVE PARTITIONING.""" - raise UnsupportedFeatureError( - self.dialect.name, "REMOVE PARTITIONING", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseCoalescePartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... COALESCE PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table, count: int): - super().__init__(dialect) - if not isinstance(count, int) or count <= 0: - raise ValueError("count must be a positive integer") - self.table = table - self.count = count - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative COALESCE PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "COALESCE PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseAnalyzePartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... ANALYZE PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) - self.table = table - self.partitions = list(partitions) - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative ANALYZE PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "ANALYZE PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseCheckPartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... CHECK PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) + """ + Args: + dialect: ClickHouse dialect instance. + table: Target table name. + partition_id: Value of ``system.parts.partition_id`` for the + partition this statement addresses. + schema: Database name, when the table is not in the connection's + current database. + + Raises: + ValueError: if ``table`` or ``partition_id`` is empty/whitespace. + TypeError: if ``partition_id`` is not a string. + """ + super().__init__(dialect) + if not isinstance(table, str) or not table.strip(): + raise ValueError("table must be a non-empty string") + if not isinstance(partition_id, str): + raise TypeError( + "partition_id must be a string (system.parts.partition_id), " + f"got {type(partition_id).__name__}" + ) + if not partition_id.strip(): + raise ValueError("partition_id must be a non-empty string") + # Stored verbatim rather than as a Table object, so that the generic + # ``get_params()`` emits exactly what the constructor accepted and the + # round trip rebuilds the same instance. The dialect wraps these into a + # ``Table`` at render time, where the database lands in the catalog + # slot -- the slot ClickHouse actually has. self.table = table - self.partitions = list(partitions) - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative CHECK PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "CHECK PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - - -class ClickHouseOptimizePartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... OPTIMIZE PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ + self.schema = schema + self.partition_id = partition_id - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) - self.table = table - self.partitions = list(partitions) + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression. - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative OPTIMIZE PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "OPTIMIZE PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", + Declared as a constant by each subclass (the convention the rest of + this backend follows, and what the structural sweep in + ``test_clickhouse_type_protocol.py`` reads off the class rather than an + instance). ``test_format_method_names_the_dialect_formatter`` in + ``tests/.../backend/expression/test_expression_signatures.py`` keeps each + constant in step with its ``verb``. + """ + raise NotImplementedError( + f"{type(self).__name__} must declare its format_method property." ) - -class ClickHouseRebuildPartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... REBUILD PARTITION`` expression. - - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. - """ - - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) - self.table = table - self.partitions = list(partitions) - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative REBUILD PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "REBUILD PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(table={self.table!r}, " + f"partition_id={self.partition_id!r})" ) -class ClickHouseRepairPartitionExpression(BaseExpression): - """MySQL ``ALTER TABLE ... REPAIR PARTITION`` expression. +class ClickHouseDropPartitionExpression(_ClickHousePartitionIdExpression): + """``ALTER TABLE ... DROP PARTITION ID`` — deletes a partition's data. - Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. + The partition's parts are tagged inactive and deleted in the background + (~10 minutes), and the statement is replicated on a replicated table. """ - def __init__(self, dialect: "ClickHouseDialect", table: Table, partitions: Sequence[str]): - super().__init__(dialect) - self.table = table - self.partitions = list(partitions) - - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: MySQL declarative REPAIR PARTITION.""" - raise UnsupportedFeatureError( - self.dialect.name, "REPAIR PARTITION", - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) - + verb = "DROP" -class ClickHousePartitionNameListExpression(BaseExpression): - """ClickHouse partition name list expression. - - Represents a list of partition names for use in partition-related operations. - This expression follows the format signature compliance pattern where the - expression object encapsulates the partition names and the dialect's - ``format_partition_name_list`` method renders it. + @property + def format_method(self) -> str: + return "format_drop_partition_statement" - ClickHouse does not support MySQL declarative partition name lists; - this expression is retained for interface compatibility. The dialect's - formatting method raises ``UnsupportedFeatureError``. - Args: - dialect: ClickHouse dialect instance. - partitions: List of partition names (strings). +class ClickHouseDetachPartitionExpression(_ClickHousePartitionIdExpression): + """``ALTER TABLE ... DETACH PARTITION ID`` — moves a partition aside. - Raises: - TypeError: if partitions is not a sequence of strings. - ValueError: if partitions is empty. + Unlike ``DROP``, the data survives in the table's ``detached/`` directory, + so it can be inspected on disk and put back with + :class:`ClickHouseAttachPartitionExpression`. The server forgets the + partition until it is attached again. """ - def __init__(self, dialect: "ClickHouseDialect", partitions: Sequence[str]): - super().__init__(dialect) - if not partitions: - raise ValueError("partitions must not be empty") - if not isinstance(partitions, AbcSequence): - partitions = list(partitions) - for i, p in enumerate(partitions): - if not isinstance(p, str): - raise TypeError( - f"partition name at index {i} must be a string, " - f"got {type(p).__name__}" - ) - self.partitions: List[str] = list(partitions) + verb = "DETACH" @property def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_partition_name_list" - + return "format_detach_partition_statement" -class ClickHouseGetPartitionsExpression(BaseExpression): - """MySQL ``information_schema.PARTITIONS`` query expression. - This is MySQL declarative partition introspection, not supported by - ClickHouse. ClickHouse partition introspection uses the ``system.parts`` - table. Retained for interface compatibility; ``to_sql()`` raises - ``UnsupportedFeatureError``. +class ClickHouseAttachPartitionExpression(_ClickHousePartitionIdExpression): + """``ALTER TABLE ... ATTACH PARTITION ID`` — re-adds a detached partition. - Raises: - TypeError: ``table`` is not a Table object. + Reads the partition back from the ``detached/`` directory and returns its + rows to the table; the inverse of + :class:`ClickHouseDetachPartitionExpression`. """ - def __init__(self, dialect: "ClickHouseDialect", table: Table): - super().__init__(dialect) - # The parameter is a Table object, and always has been: the rest of this - # class names it ``table: Table`` and every sibling takes the same. The - # emptiness test called ``table.strip()``, which a Table does not have, - # so the one thing this constructor claimed to check could not be - # reached with any valid argument -- which is what left the class - # unconstructible, and so untested. A bare string is refused instead: - # it cannot say which database it lives in, and the database is the only - # namespace there is. - if not isinstance(table, Table): - raise TypeError( - "table must be a Table object carrying its own catalog_name, " - f"got {type(table).__name__}" - ) - self.table = table + verb = "ATTACH" - def to_sql(self) -> SQLQueryAndParams: - """Raise UnsupportedFeatureError: use ``system.parts`` for introspection.""" - raise UnsupportedFeatureError( - self.dialect.name, "information_schema.PARTITIONS introspection", - suggestion="ClickHouse partition introspection uses the system.parts " - "table, not MySQL information_schema.PARTITIONS.", - ) + @property + def format_method(self) -> str: + return "format_attach_partition_statement" \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition_lifecycle.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition_lifecycle.py deleted file mode 100644 index f171381..0000000 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition_lifecycle.py +++ /dev/null @@ -1,222 +0,0 @@ -# src/rhosocial/activerecord/backend/impl/clickhouse/expression/partition_lifecycle.py -"""ClickHouse partition lifecycle management helpers. - -This module provides named expression helpers that compose the low-level -DDL expressions from :mod:`partition` into common partition management -operations: add, drop, coalesce, reorganize, and add subpartition. - -All methods return ``BaseExpression`` subclasses ready for ``to_sql()`` -or for use in a statement execution pipeline. -""" - -from typing import List, Optional, Sequence, TYPE_CHECKING, Union - -from rhosocial.activerecord.backend.expression.bases import BaseExpression, SQLQueryAndParams -from rhosocial.activerecord.backend.expression.objects import Table -from .partition import ( - ClickHouseAddPartitionExpression, - ClickHouseCoalescePartitionExpression, - ClickHouseDropPartitionExpression, - ClickHousePartitionDefinition, - ClickHousePartitionValue, - ClickHouseReorganizePartitionExpression, - ClickHouseSubpartitionDefinition, -) - -if TYPE_CHECKING: # pragma: no cover - from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect - - -class ClickHouseAddPartitionHelper(BaseExpression): - """Helper that adds multiple partitions with generated names. - - .. code-block:: python - - expr = ClickHouseAddPartitionHelper( - dialect, table=Table(dialect, "orders"), - partition_values=[2000, 2001, 2002], - name_template="p{value}", - ) - - Args: - dialect: ClickHouse dialect instance. - table: The table the partitions belong to. - partition_values: Values for ``VALUES LESS THAN`` of each new partition. - name_template: Format string for partition names (default ``"p{value}"``). - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partition_values: Sequence[Union[int, str]], - name_template: str = "p{value}", - ): - super().__init__(dialect) - self.table = table - if not partition_values: - raise ValueError("partition_values must not be empty") - self.partition_values = list(partition_values) - self.name_template = name_template - - def to_sql(self) -> SQLQueryAndParams: - partitions = [] - for value in self.partition_values: - name = self.name_template.format(value=value) - partitions.append( - ClickHousePartitionDefinition( - name=name, - less_than=[ClickHousePartitionValue(self.dialect, value)], - ) - ) - expr = ClickHouseAddPartitionExpression(self.dialect, self.table, partitions) - return expr.to_sql() - - -class ClickHouseCoalescePartitionHelper(BaseExpression): - """Helper that coalesces partitions with validation. - - Args: - dialect: ClickHouse dialect instance. - table: The table the partitions belong to. - target_count: Desired number of partitions after coalescing. - current_count: Current number of partitions (for validation). - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - target_count: int, - current_count: int, - ): - super().__init__(dialect) - self.table = table - if target_count <= 0: - raise ValueError("target_count must be positive") - self.target_count = target_count - self.current_count = current_count - - def to_sql(self) -> SQLQueryAndParams: - if self.target_count >= self.current_count: - raise ValueError( - f"target_count ({self.target_count}) must be less than " - f"current_count ({self.current_count})" - ) - count = self.current_count - self.target_count - expr = ClickHouseCoalescePartitionExpression(self.dialect, self.table, count) - return expr.to_sql() - - -class ClickHouseDropOldestPartitionHelper(BaseExpression): - """Helper that drops the oldest partition from a list of partition names. - - Sorts partitions by name and drops the first one (lexicographically oldest). - - Args: - dialect: ClickHouse dialect instance. - table: The table the partitions belong to. - partition_names: List of existing partition names. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partition_names: Sequence[str], - ): - super().__init__(dialect) - self.table = table - if not partition_names: - raise ValueError("partition_names must not be empty") - self.partition_names = list(partition_names) - - def to_sql(self) -> SQLQueryAndParams: - sorted_names = sorted(self.partition_names) - expr = ClickHouseDropPartitionExpression(self.dialect, self.table, [sorted_names[0]]) - return expr.to_sql() - - -class ClickHouseReorganizePartitionHelper(BaseExpression): - """Helper that reorganizes a partition into multiple new partitions. - - Args: - dialect: ClickHouse dialect instance. - table: The table the partitions belong to. - partition: Existing partition name to reorganize. - into: Definitions for the new partitions. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partition: str, - into: List[ClickHousePartitionDefinition], - ): - super().__init__(dialect) - self.table = table - self.partition = partition - self.into = into - - def to_sql(self) -> SQLQueryAndParams: - expr = ClickHouseReorganizePartitionExpression( - self.dialect, self.table, self.partition, self.into - ) - return expr.to_sql() - - -class ClickHouseAddSubpartitionHelper(BaseExpression): - """Helper that adds a partition with explicit subpartition definitions. - - Useful when a table already has a ``SUBPARTITION BY`` clause and you - need to add a new partition together with its subpartitions. - - Args: - dialect: ClickHouse dialect instance. - table: The table the partitions belong to. - partition_name: Name for the new partition. - less_than: ``VALUES LESS THAN`` bound(s) for the partition. - subpartition_names: Names for the subpartitions. - in_values: Optional ``VALUES IN`` values for LIST-based partitioning. - """ - - def __init__( - self, - dialect: "ClickHouseDialect", - table: Table, - partition_name: str, - less_than: Optional[Sequence] = None, - subpartition_names: Optional[Sequence[str]] = None, - in_values: Optional[Sequence] = None, - ): - super().__init__(dialect) - self.table = table - self.partition_name = partition_name - self.less_than = list(less_than) if less_than else None - self.in_values = list(in_values) if in_values else None - self.subpartition_names = list(subpartition_names) if subpartition_names else None - - def to_sql(self) -> SQLQueryAndParams: - sub_defs = None - if self.subpartition_names: - sub_defs = [ - ClickHouseSubpartitionDefinition(name=sn) - for sn in self.subpartition_names - ] - - kwargs = {"name": self.partition_name, "subpartition_definitions": sub_defs} - if self.less_than is not None: - kwargs["less_than"] = [ - ClickHousePartitionValue(self.dialect, v) if not isinstance(v, BaseExpression) else v - for v in self.less_than - ] - if self.in_values is not None: - kwargs["in_values"] = [ - ClickHousePartitionValue(self.dialect, v) if not isinstance(v, BaseExpression) else v - for v in self.in_values - ] - - definition = ClickHousePartitionDefinition(**kwargs) - expr = ClickHouseAddPartitionExpression(self.dialect, self.table, [definition]) - return expr.to_sql() diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/show.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/show.py index 10c2fda..79fd27b 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/show.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/show.py @@ -15,11 +15,19 @@ - to_sql() delegates to dialect.format_show_* methods - Dialect handles actual SQL generation -.. warning:: - This module was copied from the MySQL backend template and contains - MySQL-style SQL functions/show commands. ClickHouse uses different - function names (e.g. ``JSONExtract*``) and a different SHOW command - subset. May generate non-ClickHouse SQL; verify before use. +Nine of these are ClickHouse statements, and their output column names differ +from MySQL's: ``SHOW CREATE TABLE`` / ``SHOW CREATE VIEW`` answer a single +``statement`` column; ``SHOW TABLES`` names it ``name``, adding ``engine`` +under ``FULL``; ``SHOW DATABASES`` names it ``name``; ``SHOW COLUMNS`` reports +``field`` / ``type`` / ``null`` / ``key`` / ``default`` / ``extra``; ``SHOW +INDEX`` reports ``pk_col`` where MySQL says ``Column_name``; ``SHOW +PROCESSLIST`` and ``SHOW ENGINES`` are ``system.processes`` and +``system.table_engines`` verbatim; and ``SHOW GRANTS`` returns one column named +after the statement text plus the requested output format. The rest of the +MySQL ``SHOW`` set is ClickHouse's to not have, and each of those expressions +carries a NOTE saying so — their dialect formatter raises +``UnsupportedFeatureError`` naming the ClickHouse replacement, so no expression +here renders SQL the server would reject. """ from typing import Any, Dict, Optional, TYPE_CHECKING @@ -104,12 +112,9 @@ def to_sql(self) -> SQLQueryAndParams: class ShowColumnsExpression(ShowExpression): """Expression for SHOW [FULL] COLUMNS command. - NOTE: MySQL-only command, not supported by ClickHouse. - - ``full``, ``like_pattern`` and ``schema`` are keyword arguments as well as - fluent methods, for the reason given on :class:`ShowTablesExpression`: the - constructor has to accept everything ``get_params`` emits, or the expression - cannot be rebuilt from its own serialization. + ClickHouse has this statement; the table is mandatory (bare ``SHOW COLUMNS`` + is a ``SYNTAX_ERROR``) and the database, when given, is part of the table + reference (``SHOW COLUMNS FROM db.t``), never a second ``FROM``. """ def __init__( @@ -153,7 +158,8 @@ def to_sql(self) -> SQLQueryAndParams: class ShowIndexExpression(ShowExpression): """Expression for SHOW INDEX command. - NOTE: MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement "mostly for compatibility with MySQL". As with + ``SHOW COLUMNS``, the database is part of the table reference. """ def __init__(self, dialect: "ClickHouseDialect", table: str): @@ -410,11 +416,9 @@ def to_sql(self) -> SQLQueryAndParams: class ShowProcessListExpression(ShowExpression): """Expression for SHOW PROCESSLIST command. - NOTE: MySQL-only command, not supported by ClickHouse. - - ``full`` and ``schema`` are keyword arguments as well as fluent methods; see - :class:`ShowTablesExpression` for why the constructor has to accept what - ``get_params`` emits. + ClickHouse has this statement; it is ``SELECT * FROM system.processes``. + ``full`` is accepted and never rendered — the statement has no ``FULL`` + variant. """ def __init__( @@ -496,7 +500,7 @@ def to_sql(self) -> SQLQueryAndParams: class ShowEnginesExpression(ShowExpression): """Expression for SHOW ENGINES command. - NOTE: MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement; it is ``SELECT * FROM system.table_engines``. """ def to_sql(self) -> SQLQueryAndParams: @@ -556,7 +560,8 @@ def to_sql(self) -> SQLQueryAndParams: class ShowGrantsExpression(ShowExpression): """Expression for SHOW GRANTS command. - NOTE: MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement. Its ``FOR`` clause takes access-entity names + only — there is no ``user@host`` form — so ``host`` is accepted and ignored. """ def __init__(self, dialect: "ClickHouseDialect"): diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/table_statement.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/table_statement.py index 895497d..d8c5908 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/table_statement.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/table_statement.py @@ -1,13 +1,39 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/expression/table_statement.py -"""ClickHouse TABLE statement and VALUES table-value constructor expressions. +"""MySQL's ``TABLE`` / ``VALUES`` simplified statements — ClickHouse has neither. -ClickHouse 8.0+ supports two simplified query forms: +MySQL 8.0.19 added two query forms:: TABLE [ORDER BY ...] [LIMIT ...] VALUES ROW(...), ROW(...) [ORDER BY ...] [LIMIT ...] -``TABLE`` is a shortcut for ``SELECT * FROM
`` and ``VALUES`` lets a -row list be used as a table value constructor. +where ``TABLE`` is a shortcut for ``SELECT * FROM
`` and ``VALUES`` is a +row-list table-value constructor. **ClickHouse has no such statements at any +version**, so there is no version floor to name. On 26.7.3.19: + +* ``TABLE system.metrics`` — ``Code: 62. DB::Exception: Syntax error: failed at + position 1 (TABLE): TABLE system.metrics. Expected one of: Query, Query with + output, EXPLAIN, EXPLAIN, SELECT query, possibly with UNION, list of union + elements, SELECT query, subquery, possibly with UNION, SELECT or EXPLAIN + subquery, SELECT query, WITH, FROM, SELECT, SHOW CREATE QUOTA query, SHOW + CREATE, SHOW [FULL] [TEMPORARY] TABLES|DATABASES|CLUSTERS|CLUSTER|MERGES + 'name' [[NOT] [I]LIKE 'str'] [LIMIT expr], SHOW, SHOW COLUMNS query, SHOW + ENGINES query, ... (SYNTAX_ERROR)``. That is the server listing every + top-level statement it accepts, and ``TABLE`` is not among them — nor is + ``VALUES``. +* ``SELECT * FROM VALUES(1,2,3)`` **does** work, and + ``system.table_functions`` lists ``values``. But that is the ``values`` + *table function* taking a comma-separated list of scalar values, not MySQL's + ``VALUES ROW(...), ROW(...)`` statement: ``SELECT * FROM + VALUES(ROW(1,2), ROW(3,4))`` is ``Code: 46. DB::Exception: Function with name + `ROW` does not exist. In scope SELECT * FROM `VALUES`(ROW(1, 2), ROW(3, 4)). + Maybe you meant: ['now','pow']. (UNKNOWN_FUNCTION)`` — ClickHouse has no + ``ROW`` constructor function at all. + +So both expression classes below are fail-fast switches: +``supports_table_statement()`` and ``supports_values_table_constructor()`` are +``False``, and ``format_table_statement`` / ``format_values_statement`` raise +``UnsupportedFeatureError`` naming ``SELECT * FROM
`` and +``SELECT ... UNION ALL SELECT ...`` as the ClickHouse spellings. """ from typing import Any, List, Optional, TYPE_CHECKING @@ -48,7 +74,12 @@ def _validate_common(self) -> None: class ClickHouseTableExpression(ClickHouseBaseTableStatement): - """Represent the ClickHouse ``TABLE
`` simplified SELECT statement.""" + """MySQL's ``TABLE
`` simplified SELECT — a statement ClickHouse does not have. + + Fail-fast switch: ``to_sql()`` raises ``UnsupportedFeatureError`` through the + dialect, naming ``SELECT * FROM
``. No ClickHouse release parses it — + see the module docstring. + """ def __init__( self, @@ -80,7 +111,14 @@ def format_method(self) -> str: class ClickHouseValuesExpression(ClickHouseBaseTableStatement): - """Represent the ClickHouse ``VALUES ROW(...), ...`` table value constructor.""" + """MySQL's ``VALUES ROW(...), ...`` table value constructor — not in ClickHouse. + + Fail-fast switch: ``to_sql()`` raises ``UnsupportedFeatureError`` through the + dialect, naming ``SELECT ... UNION ALL SELECT ...``. ClickHouse's ``values`` + *table function* is a different thing and is not what this models — it takes + bare scalar values and there is no ``ROW()`` constructor to pass it; see the + module docstring. + """ def __init__( self, diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/types.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/types.py index 5ccf778..b6fd5ca 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/types.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/types.py @@ -13,90 +13,250 @@ introspection result parsing, and schema comparison. They should **not** be used by application code directly — always use the core types for DDL definition expressions (``ColumnDefinition.data_type``). + +Inheritance +----------- +Every ClickHouse type that *is* a core concept derives from the core class it +is, rather than sitting on ``DataType`` and restating it: + +============================ ====================== +ClickHouse type Core concept +============================ ====================== +``Int8`` / ``UInt8`` :class:`TinyIntType` +``Int16`` / ``UInt16`` :class:`SmallIntType` +``Int32`` / ``UInt32`` :class:`IntegerType` +``Int64`` / ``UInt64`` :class:`BigIntType` +``Float32`` :class:`FloatType` +``Float64`` :class:`DoubleType` +``Decimal*`` :class:`DecimalType` +``String`` :class:`TextType` +``FixedString(N)`` :class:`BinaryType` +``Date`` / ``Date32`` :class:`DateType` +``DateTime`` / ``DateTime64`` :class:`TimestampType` +``Bool`` :class:`BooleanType` +``UUID`` :class:`UUIDType` +``JSON`` :class:`JsonType` +============================ ====================== + +The ``name`` stays ``clickhouse_``-prefixed in every case, because the DDL really +does say ``Int32`` and a schema diff has to compare the word the server wrote. +Inheritance says what the type *is*; ``name`` says how it is *spelled*. + +What is left on ``DataType`` is what ClickHouse has and core does not — the +wrappers, the containers, the enum, the network addresses, the spatial family +and the vector type. Each of those carries a docstring saying why it is not a +core concept. """ from __future__ import annotations from typing import List, Optional, Tuple as TupleType -from rhosocial.activerecord.backend.expression.types import DataType +from rhosocial.activerecord.backend.expression.types import ( + BigIntType, + BinaryType, + BooleanType, + DataType, + DateType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + JsonType, + SmallIntType, + TextType, + TimestampType, + TinyIntType, + UUIDType, +) from rhosocial.activerecord.backend.expression.types.array import ArrayType # --------------------------------------------------------------------------- # Integer types (signed and unsigned) +# +# ClickHouse is the one backend where width and signedness are two independent +# axes over the same eight storage sizes: ``Int8``..``Int64`` and +# ``UInt8``..``UInt64``. Core already models that shape — width is the class, +# signedness is the ``unsigned`` field — so each ClickHouse integer is the core +# width class whose storage it uses, and nothing else. # --------------------------------------------------------------------------- -class ClickHouseInt8Type(DataType): - """ClickHouse ``Int8`` — signed 8-bit integer.""" +class ClickHouseInt8Type(TinyIntType): + """ClickHouse ``Int8`` — signed 8-bit integer, range [-128, 127]. + + The 1-byte signed integer concept. ``spelling`` stays a core word + (``tinyint``): ClickHouse has no spelling of its own for it, ``Int8`` is the + name of the *column*, not of the concept. + """ name = "clickhouse_int8" -class ClickHouseInt16Type(DataType): - """ClickHouse ``Int16`` — signed 16-bit integer.""" +class ClickHouseInt16Type(SmallIntType): + """ClickHouse ``Int16`` — signed 16-bit integer, range [-32768, 32767]. + + The 2-byte signed integer concept; see :class:`ClickHouseInt8Type`. + """ name = "clickhouse_int16" -class ClickHouseInt32Type(DataType): - """ClickHouse ``Int32`` — signed 32-bit integer.""" +class ClickHouseInt32Type(IntegerType): + """ClickHouse ``Int32`` — signed 32-bit integer. + + The 4-byte signed integer concept, and the default integer ClickHouse uses + for a number with no width stated. + """ name = "clickhouse_int32" -class ClickHouseInt64Type(DataType): - """ClickHouse ``Int64`` — signed 64-bit integer.""" +class ClickHouseInt64Type(BigIntType): + """ClickHouse ``Int64`` — signed 64-bit integer. + + The 8-byte signed integer concept. ClickHouse also uses ``Int64`` as the + storage for an interval expressed as a count of seconds, which is why + ``suggested_data_types()`` names this class as the stand-in for core's + ``IntervalType``. + """ name = "clickhouse_int64" -class ClickHouseUInt8Type(DataType): - """ClickHouse ``UInt8`` — unsigned 8-bit integer (also used for Boolean).""" +class ClickHouseUInt8Type(TinyIntType): + """ClickHouse ``UInt8`` — unsigned 8-bit integer, range [0, 255]. + + Same width concept as ``Int8`` but the opposite value range, and ClickHouse + spells that in the name rather than as a modifier, so it is a separate class + with ``unsigned`` pinned on rather than a flag a caller can flip. + + ``UInt8`` is also the physical type behind ``Bool``; that is + :class:`ClickHouseBoolType`, kept separate because the two do not compare or + convert the same way. + """ name = "clickhouse_uint8" + def __init__(self, dialect=None): + super().__init__(dialect, unsigned=True) + -class ClickHouseUInt16Type(DataType): - """ClickHouse ``UInt16`` — unsigned 16-bit integer.""" +class ClickHouseUInt16Type(SmallIntType): + """ClickHouse ``UInt16`` — unsigned 16-bit integer, range [0, 65535]. + + See :class:`ClickHouseUInt8Type` for why this is a class and not a flag. + """ name = "clickhouse_uint16" + def __init__(self, dialect=None): + super().__init__(dialect, unsigned=True) + + +class ClickHouseUInt32Type(IntegerType): + """ClickHouse ``UInt32`` — unsigned 32-bit integer, range [0, 4294967295]. -class ClickHouseUInt32Type(DataType): - """ClickHouse ``UInt32`` — unsigned 32-bit integer.""" + See :class:`ClickHouseUInt8Type` for why this is a class and not a flag. + """ name = "clickhouse_uint32" + def __init__(self, dialect=None): + super().__init__(dialect, unsigned=True) -class ClickHouseUInt64Type(DataType): - """ClickHouse ``UInt64`` — unsigned 64-bit integer.""" + +class ClickHouseUInt64Type(BigIntType): + """ClickHouse ``UInt64`` — unsigned 64-bit integer. + + See :class:`ClickHouseUInt8Type` for why this is a class and not a flag. + """ name = "clickhouse_uint64" + def __init__(self, dialect=None): + super().__init__(dialect, unsigned=True) + # --------------------------------------------------------------------------- # Float types # --------------------------------------------------------------------------- -class ClickHouseFloat32Type(DataType): - """ClickHouse ``Float32`` — 32-bit floating point.""" +class ClickHouseFloat32Type(FloatType): + """ClickHouse ``Float32`` — 32-bit floating point. + + The variable-precision approximate-numeric concept, but ClickHouse has no + adjustable precision to give it: ``Float32`` is 32 bits and that is the whole + type, where ``FLOAT(p)`` in the generic spelling would choose a mantissa + width. + + So the class declares **no** parameters. ``PARAMETERS`` and ``__init__`` are + both stated for the same reason — ``precision`` is not a constructor argument + at all, because accepting one would let a caller set a width the server will + silently ignore, and because :meth:`BaseExpression.get_params` reads the + constructor signature, so the two must agree about what the parameters are. + """ + + # ClickHouse's ``Float32`` takes no precision argument -- ``__init__`` rejects + # one -- so declaring ``precision`` here would advertise an identity the + # class cannot have. Declared empty deliberately. + + PARAMETERS = () name = "clickhouse_float32" + def __init__(self, dialect=None): + super().__init__(dialect) + + + super().__init__(dialect) + +class ClickHouseFloat64Type(DoubleType): + """ClickHouse ``Float64`` — 64-bit floating point. -class ClickHouseFloat64Type(DataType): - """ClickHouse ``Float64`` — 64-bit floating point.""" + Core's ``DoubleType`` is ``DOUBLE PRECISION``; ClickHouse's word for the same + 8-byte float is ``Float64``. The concept is the same — an approximate number + with a 53-bit mantissa — so the class derives from it, and declares no + parameters for symmetry with :class:`ClickHouseFloat32Type`: ``spelling`` is + a SQL-standard word (``double`` / ``double precision``) that ClickHouse never + writes, so it is neither part of this type's identity nor something a caller + should be able to set. Both ``PARAMETERS`` and ``__init__`` are stated so + the two agree — :meth:`BaseExpression.get_params` reads the constructor + signature, and an inherited ``spelling`` would make it report a parameter + equality says does not exist. + """ + + # As ``ClickHouseFloat32Type``: no precision argument, so no precision in + # identity. + + PARAMETERS = () name = "clickhouse_float64" + def __init__(self, dialect=None): + super().__init__(dialect) + # --------------------------------------------------------------------------- # Decimal types +# +# ClickHouse has four ways to spell an exact fixed-point number, and the choice +# is not cosmetic: ``Decimal(P, S)`` takes an explicit precision, while +# ``Decimal32`` / ``Decimal64`` / ``Decimal128`` fix it at 9 / 18 / 38 digits and +# take only the scale. A schema diff has to see that difference, which is why +# these are four classes over one core concept rather than one class with a +# precision. # --------------------------------------------------------------------------- -class ClickHouseDecimalType(DataType): - """ClickHouse ``Decimal(P, S)`` — fixed-point number.""" +class ClickHouseDecimalType(DecimalType): + """ClickHouse ``Decimal(P, S)`` — exact fixed-point with explicit precision. + + ClickHouse caps the precision at 38 digits, tighter than the SQL standard's + 38 (and much tighter than PostgreSQL's 1000), so the bound is enforced here + rather than left to the server. + """ + + PARAMETERS = ("precision", "scale",) name = "clickhouse_decimal" @@ -116,12 +276,15 @@ def __init__(self, dialect=None, *, precision: int, scale: int = 0): self.precision = precision self.scale = scale - def _type_params(self) -> tuple: - return (self.precision, self.scale) +class ClickHouseDecimal32Type(DecimalType): + """ClickHouse ``Decimal32(S)`` — exact fixed-point in 32 bits, 9 digits. -class ClickHouseDecimal32Type(DataType): - """ClickHouse ``Decimal32(S)`` — 32-bit decimal with 9 digits.""" + The precision is fixed by the width, so it is the class that carries it and + only the scale is a parameter — which is why ``precision`` deliberately does + not appear in ``PARAMETERS``: two ``Decimal32`` columns with the same scale + are the same type whatever the class docstring says about digit counts. + """ name = "clickhouse_decimal32" @@ -135,12 +298,13 @@ def __init__(self, dialect=None, *, scale: int = 0): ) self.scale = scale - def _type_params(self) -> tuple: - return (self.scale,) + PARAMETERS = ("scale",) +class ClickHouseDecimal64Type(DecimalType): + """ClickHouse ``Decimal64(S)`` — exact fixed-point in 64 bits, 18 digits. -class ClickHouseDecimal64Type(DataType): - """ClickHouse ``Decimal64(S)`` — 64-bit decimal with 18 digits.""" + See :class:`ClickHouseDecimal32Type` on why only the scale is a parameter. + """ name = "clickhouse_decimal64" @@ -154,12 +318,13 @@ def __init__(self, dialect=None, *, scale: int = 0): ) self.scale = scale - def _type_params(self) -> tuple: - return (self.scale,) + PARAMETERS = ("scale",) +class ClickHouseDecimal128Type(DecimalType): + """ClickHouse ``Decimal128(S)`` — exact fixed-point in 128 bits, 38 digits. -class ClickHouseDecimal128Type(DataType): - """ClickHouse ``Decimal128(S)`` — 128-bit decimal with 38 digits.""" + See :class:`ClickHouseDecimal32Type` on why only the scale is a parameter. + """ name = "clickhouse_decimal128" @@ -173,22 +338,54 @@ def __init__(self, dialect=None, *, scale: int = 0): ) self.scale = scale - def _type_params(self) -> tuple: - return (self.scale,) - + PARAMETERS = ("scale",) # --------------------------------------------------------------------------- # String / Binary types +# +# ClickHouse has exactly two string types and they divide on padding, not on +# character set. ``String`` is variable-length and stores the bytes as given; +# ``FixedString(N)`` is exactly N **bytes**, right-padded with ``\\0`` to length +# N and rejecting anything longer. That padding is what makes the second one a +# byte string: a fixed-length *character* type cannot behave that way, and +# ClickHouse's own documentation shows ``FixedString(3)`` holding ``'a'`` +# matching ``= 'a'`` while ``LIKE 'a'`` matches nothing, because the stored value +# is ``'a\\0\\0'``. # --------------------------------------------------------------------------- -class ClickHouseStringType(DataType): - """ClickHouse ``String`` — variable-length binary/string.""" +class ClickHouseStringType(TextType): + """ClickHouse ``String`` — variable-length byte string, no padding. + + The unbounded-text concept: ClickHouse's manual recommends it for UTF-8 text + and appends nothing, so what is stored is what was written. It is *not* a + ``CHAR(n)`` — ClickHouse has no fixed-width character type at all, which is + why ``CharType`` and ``VarCharType`` both render ``String`` here and the + declared length is dropped. + """ name = "clickhouse_string" -class ClickHouseFixedStringType(DataType): - """ClickHouse ``FixedString(N)`` — fixed-length binary/string.""" +class ClickHouseFixedStringType(BinaryType): + """ClickHouse ``FixedString(N)`` — exactly N bytes, zero-padded on the right. + + The fixed-length **byte** string concept, not the fixed-length character + one. Two properties make that the honest reading and not a matter of taste: + + * ``N`` counts bytes, not characters, so a UTF-8 column is limited by bytes + and a character can be split by the padding; + * a shorter value is padded with ``\\0`` to exactly N, which is why + ``FixedString(3)`` holding ``'a'`` equals ``'a'`` but does not match + ``LIKE 'a'`` — a fixed-length character type cannot do that. + + ``length`` is required here although :class:`BinaryType` makes it optional, + because ``FixedString`` with no length is not a type ClickHouse will accept. + """ + + # ``FixedString(N)`` is exactly N zero-padded bytes, so N is not a hint but + # the storage: a FixedString(4) and a FixedString(8) are different columns. + + PARAMETERS = ("length",) name = "clickhouse_fixed_string" @@ -200,36 +397,92 @@ def __init__(self, dialect=None, *, length: int): raise ValueError("FixedString length must be >= 1") self.length = length - def _type_params(self) -> tuple: - return (self.length,) - # --------------------------------------------------------------------------- # Date / Time types +# +# ClickHouse's ``DateTime`` is a count of seconds since the Unix epoch — the +# thing SQL calls ``TIMESTAMP``, not MySQL's wall-clock ``DATETIME``. That is why +# both ``ClickHouseDateTimeType`` and ``ClickHouseDateTime64Type`` derive from +# :class:`TimestampType` even though one of them is spelled ``DateTime``. # --------------------------------------------------------------------------- -class ClickHouseDateType(DataType): - """ClickHouse ``Date`` — date (2 bytes, 1970-2149).""" +class ClickHouseDateType(DateType): + """ClickHouse ``Date`` — calendar date in 2 bytes, 1970-01-01..2149-06-06. + + The date concept at its narrowest. :class:`ClickHouseDate32Type` is the same + concept in 4 bytes over a wider range, which is a storage difference a schema + diff must notice, so it is a second class over the same core concept. + """ name = "clickhouse_date" -class ClickHouseDate32Type(DataType): - """ClickHouse ``Date32`` — extended date (4 bytes, 1900-2299).""" +class ClickHouseDate32Type(DateType): + """ClickHouse ``Date32`` — calendar date in 4 bytes, 1900-01-01..2299-12-31. + + Same concept as :class:`ClickHouseDateType`, four times the range, and a + different type to ClickHouse — a column cannot change from one to the other + in place without rewriting what it holds. + """ name = "clickhouse_date32" -class ClickHouseDateTimeType(DataType): - """ClickHouse ``DateTime`` — date and time (4 bytes, seconds precision).""" +class ClickHouseDateTimeType(TimestampType): + """ClickHouse ``DateTime`` — seconds since the Unix epoch, no sub-second part. + + Core's ``TimestampType`` is ``TIMESTAMP[(p)]``: a point on the timeline, + stored as an epoch count and rendered in a session or column time zone. That + is exactly what ClickHouse's ``DateTime`` is, so it is the same concept — and + specifically *not* core's ``DateTimeType``, which is MySQL's wall-clock + ``DATETIME`` with no epoch to convert against. + + The precision is fixed at 0 (whole seconds) and there is nothing to set, so + this declares **no** parameters: ``precision`` is not a constructor argument, + because accepting one would promise a sub-second precision the server will + not keep, and :meth:`BaseExpression.get_params` reads the constructor + signature, so it and ``PARAMETERS`` have to agree. + :class:`ClickHouseDateTime64Type` is the adjustable-precision variant. + + The dispatch key is ``clickhouse_timestamp``, **not** + ``clickhouse_datetime``, even though the server's word is ``DateTime``. The + key names the concept; the renderer writes ClickHouse's spelling of it — + exactly as this backend writes ``String`` for a text column. An earlier key + of ``clickhouse_datetime`` said "datetime" while the class derived from + ``TimestampType``: a reader comparing the key against core's type list would + conclude the base was wrong, and act on it. + """ + + # As the Float classes: ``__init__`` takes no precision, so inheriting + # ``precision`` from the timestamp concept would advertise an identity + # this class cannot have. Whole seconds only. + PARAMETERS = () + + name = "clickhouse_timestamp" + + def __init__(self, dialect=None): + super().__init__(dialect) + - name = "clickhouse_datetime" +class ClickHouseDateTime64Type(TimestampType): + """ClickHouse ``DateTime64(P)`` — epoch seconds with 0..9 fractional digits. + The same concept as :class:`ClickHouseDateTimeType` at a chosen precision, + stored as a scaled integer. ``precision`` is a real parameter here, so + ``PARAMETERS`` is stated rather than inherited only to make it obvious that + it is the precision — and not some other inherited field — that makes two + ``DateTime64`` columns different types. -class ClickHouseDateTime64Type(DataType): - """ClickHouse ``DateTime64(P)`` — date and time with sub-second precision.""" + The dispatch key is ``clickhouse_timestamp64``, not + ``clickhouse_datetime64``, even though the server's word is ``DateTime64``. + The key names the *concept*; the renderer writes ClickHouse's spelling. The + old key said "datetime" while the class derived from ``TimestampType``, which + is the one claim a reader would act on and get wrong. See + :class:`ClickHouseDateTimeType` for why the concept is a timestamp here. + """ - name = "clickhouse_datetime64" + name = "clickhouse_timestamp64" precision: int @@ -239,16 +492,23 @@ def __init__(self, dialect=None, *, precision: int = 3): raise ValueError("DateTime64 precision must be between 0 and 9") self.precision = precision - def _type_params(self) -> tuple: - return (self.precision,) - + PARAMETERS = ("precision",) # --------------------------------------------------------------------------- # Boolean type # --------------------------------------------------------------------------- -class ClickHouseBoolType(DataType): - """ClickHouse ``Bool`` — Boolean type (actually UInt8).""" +class ClickHouseBoolType(BooleanType): + """ClickHouse ``Bool`` — truth value, stored as a ``UInt8`` restricted to 0/1. + + Core's ``BooleanType`` is ``BOOLEAN`` / ``BOOL``, and ClickHouse accepts both + words for exactly this type — so the core spelling list is not a widening + here, it is ClickHouse's own vocabulary. + + ``UInt8`` is the physical type underneath, but ``Bool`` is not an integer: + ClickHouse rejects any other value on insert, which is the property + :class:`ClickHouseUInt8Type` does not have. + """ name = "clickhouse_bool" @@ -257,24 +517,55 @@ class ClickHouseBoolType(DataType): # UUID # --------------------------------------------------------------------------- -class ClickHouseUUIDType(DataType): - """ClickHouse ``UUID`` — universally unique identifier.""" +class ClickHouseUUIDType(UUIDType): + """ClickHouse ``UUID`` — 128-bit identifier, stored as two 64-bit halves. + + ClickHouse is the reference implementation for this concept on this side of + the project: the value is a plain 128-bit number with no object identity + beyond the bits, and the type is named exactly as core names it. Deriving + from :class:`UUIDType` is what lets code written against the core concept + work on this backend unchanged, and what makes "is this a UUID column?" one + question instead of one per backend. + """ name = "clickhouse_uuid" # --------------------------------------------------------------------------- # IP types +# +# Not core concepts: SQL:2016 has no network address type, and core's type +# documentation promises none. ClickHouse stores an address as its own type with +# its own operators, so each gets a class here rather than being faked with a +# character type. # --------------------------------------------------------------------------- class ClickHouseIPv4Type(DataType): - """ClickHouse ``IPv4`` — IPv4 address.""" + """ClickHouse ``IPv4`` — an IPv4 address, stored as 4 bytes. + + ClickHouse's own type, with its own operators: arithmetic on two addresses + yields a network mask rather than a number, which is why it cannot be a + ``UInt32`` with a name. + + On ``DataType`` because SQL:2016 has no network address type. Core's own + backend documentation lists the other backends' ``PostgresInetType`` / + ``MySqlInetType`` as backend types rather than promising a core concept, and + promoting one is on the datatype-hierarchy plan's shelved list — a separate + decision, not something to settle by inheritance here. + """ name = "clickhouse_ipv4" class ClickHouseIPv6Type(DataType): - """ClickHouse ``IPv6`` — IPv6 address.""" + """ClickHouse ``IPv6`` — an IPv6 address, stored as 16 bytes. + + A separate class from :class:`ClickHouseIPv4Type` because the stored length + differs, which is what a schema diff has to notice. + + On ``DataType`` because SQL:2016 has no network address type; see + :class:`ClickHouseIPv4Type`. + """ name = "clickhouse_ipv6" @@ -284,7 +575,32 @@ class ClickHouseIPv6Type(DataType): # --------------------------------------------------------------------------- class ClickHouseEnum8Type(DataType): - """ClickHouse ``Enum8`` — 8-bit enum with explicit string-value pairs.""" + """ClickHouse ``Enum8`` — a closed set of string labels over a signed byte. + + **Not** core's :class:`~...types.enum_.EnumType`, and the differences are + behavioural, not cosmetic: + + * the value space is an explicit mapping of label to **possibly negative + signed integer**, and those integers are chosen by the caller; + * the server orders the labels **by the integer assigned to each**, not by + the order they were declared in, and rewrites the declaration to match — + ``Enum8('x' = 1, 'y' = -2)`` is reported back as ``Enum8('y' = -2, 'x' = 1)``; + * the column is **not nullable**. ``Enum8`` values cannot hold NULL; the + closest equivalent is ``Nullable(Enum8(...))``, a different type; + * most numeric and string operations are simply **undefined** on it — ClickHouse + raises rather than coercing, so an enum does not inherit the arithmetic of + the integer it happens to be stored as. + + Core's ``EnumType`` carries ``values: Tuple[str, ...]`` — labels only, no + integers, no order rule and no nullability. Deriving from it would make + ``isinstance`` claim four things about a ClickHouse enum that are all false, + so the class sits on ``DataType`` (D6). + """ + + # The label-to-integer mapping is the whole type, and ClickHouse assigns + # possibly-negative integers and orders by them. + + PARAMETERS = ("values",) name = "clickhouse_enum8" @@ -302,18 +618,25 @@ def __init__(self, dialect=None, *, values: List[TupleType[str, int]]): if pair in seen: raise ValueError(f"Duplicate Enum8 pair: {pair}") seen.add(pair) - self.values = list(values) - - def _type_params(self) -> tuple: - return (tuple(self.values),) + self.values = tuple(values) + PARAMETERS = ("values",) class ClickHouseEnum16Type(DataType): - """ClickHouse ``Enum16`` — 16-bit enum with explicit string-value pairs.""" + """ClickHouse ``Enum16`` — a closed set of string labels over a signed short. + + Identical in kind to :class:`ClickHouseEnum8Type` and identical in every one + of its reasons for not being core's ``EnumType``: explicit possibly-negative + integers, ordering by the assigned integer rather than the declaration order, + non-nullable, and undefined for most numeric and string operations. + + Separate from ``Enum8`` because the label set has to fit in a signed short + rather than a signed byte, and that is a different type to the server. + """ name = "clickhouse_enum16" - values: List[TupleType[str, int]] + PARAMETERS = ("values",) def __init__(self, dialect=None, *, values: List[TupleType[str, int]]): super().__init__(dialect) @@ -327,24 +650,40 @@ def __init__(self, dialect=None, *, values: List[TupleType[str, int]]): if pair in seen: raise ValueError(f"Duplicate Enum16 pair: {pair}") seen.add(pair) - self.values = list(values) - - def _type_params(self) -> tuple: - return (tuple(self.values),) + self.values = tuple(values) + PARAMETERS = ("values",) # --------------------------------------------------------------------------- # Container types (Array, Map, Tuple) # --------------------------------------------------------------------------- class ClickHouseArrayType(ArrayType): - """ClickHouse ``Array(T)`` — array of elements.""" + """ClickHouse ``Array(T)`` — a variable-length list of ``T``. + + Already the right shape: core's :class:`~...types.array.ArrayType` is + ``T[]`` with an element type and a dimensionality, and ClickHouse's + ``Array(T)`` is the nesting spelling of it. Nesting stays available here + (``Array(Array(Int32))``) precisely because ClickHouse has no multi-dimensional + array syntax; see the core docstring on nesting versus dimensionality. + """ name = "clickhouse_array" class ClickHouseMapType(DataType): - """ClickHouse ``Map(K, V)`` — key-value map.""" + """ClickHouse ``Map(K, V)`` — a dictionary keyed by ``K``. + + The nearest SQL standard concept is ``ROW``, but a ``ROW`` is an ordered list + of *named* fields and a ``Map`` is an unordered set of key/value pairs whose + keys are values of their own type — the two accept different literals, index + differently and have different operators. Core's only container is + :class:`~...types.array.ArrayType`, which cannot express either the key type + or the lookup, so this is a ClickHouse type of its own. + + Stored as a nested ``Array(Tuple(K, V))``, which is why ClickHouse's manual + warns about its per-key overhead. + """ name = "clickhouse_map" @@ -356,12 +695,32 @@ def __init__(self, dialect=None, *, key_type: DataType, value_type: DataType): self.key_type = key_type self.value_type = value_type - def _type_params(self) -> tuple: - return (self.key_type, self.value_type) - + PARAMETERS = ("key_type", "value_type",) class ClickHouseTupleType(DataType): - """ClickHouse ``Tuple(T1, T2, ...)`` — named or unnamed tuple.""" + """ClickHouse ``Tuple(T1, T2, ...)`` — a fixed group of values. + + Elements are addressed by position (``tuple.1``) and, when the declaration + names them, by name (``tuple.a``) as well. The names **are** part of the + type's identity on this server: verified live on ClickHouse 26.7.3.19, they + are stored and reported by ``system.columns.type`` / ``SHOW CREATE`` / + ``DESCRIBE`` / ``toTypeName``, compared (and hashed) by ``IDataType::equals``, + used for ``tuple.name`` access, and matched when values convert during + INSERT/CAST. The full evidence table lives in + ``.claude/plan/2026-10-08/secondary-gaps-investigation.md`` §A. That is why + ``element_names`` sits in ``PARAMETERS``: ``Tuple(a Int32)`` and + ``Tuple(b Int32)`` are different columns, and a model declaring the latter + against a table holding the former is a change the differ must report — + the ``ALTER TABLE … MODIFY COLUMN x Tuple(b Int32)`` it emits renames the + column, where suppressing the difference would leave ``x.b`` failing at + runtime with ``UNKNOWN_IDENTIFIER``. It is not an array either: core's + :class:`~...types.array.ArrayType` has no way to say the elements are + heterogeneous. + + On ``DataType`` because SQL:2016 has no positional composite type; ``STRUCT`` + and ``ROW`` are shelved in the datatype-hierarchy plan, and a ClickHouse + ``Tuple`` is neither. + """ name = "clickhouse_tuple" @@ -375,22 +734,37 @@ def __init__(self, dialect=None, *, element_types: List[DataType], raise ValueError("Tuple must have at least one element") if element_names and len(element_names) != len(element_types): raise ValueError("element_names length must match element_types") - self.element_types = list(element_types) - self.element_names = list(element_names) if element_names else None + # Tuples from construction, so the declaration below is a plain read + # rather than a comparison that quietly normalises a list it forgot to + # normalise when the value was stored. + self.element_types = tuple(element_types) + self.element_names = tuple(element_names) if element_names else None - def _type_params(self) -> tuple: - return ( - tuple(self.element_types), - tuple(self.element_names) if self.element_names else None, - ) + PARAMETERS = ("element_types", "element_names",) # --------------------------------------------------------------------------- -# Type modifiers (Nullable, LowCardinality) +# Type wrappers (Nullable, LowCardinality) +# +# The briefing's shelved list names these four wrappers explicitly as out of +# scope for this refactor, so they keep their classes and their rendering; what +# follows is only the D7 justification for each of them sitting on ``DataType``. # --------------------------------------------------------------------------- class ClickHouseNullableType(DataType): - """ClickHouse ``Nullable(T)`` — allows NULL values for the inner type.""" + """ClickHouse ``Nullable(T)`` — ``T``, or ``NULL``. + + SQL models nullability as a **column attribute** (core's + ``ColumnDefinition.nullable``) that any type may carry; ClickHouse has no such + thing and writes it into the type name instead. That difference is not + cosmetic — a ``Nullable(String)`` column cannot hold the empty string + distinctly from ``''``, and a non-nullable one cannot hold ``NULL`` at all — + so the wrapper has to be a type in order to be rendered and compared. + + On ``DataType``: there is no core concept to derive from, because the concept + it wraps is a *modifier*, and modifiers are not in the identity of a type. + Shelved as a wrapper in the datatype-hierarchy plan. + """ name = "clickhouse_nullable" @@ -400,12 +774,23 @@ def __init__(self, dialect=None, *, inner_type: DataType): super().__init__(dialect) self.inner_type = inner_type - def _type_params(self) -> tuple: - return (self.inner_type,) - + PARAMETERS = ("inner_type",) class ClickHouseLowCardinalityType(DataType): - """ClickHouse ``LowCardinality(T)`` — dictionary-encoded type.""" + """ClickHouse ``LowCardinality(T)`` — ``T`` stored as a dictionary index. + + A **storage encoding**: the values are written into a dictionary and the + column holds positions into it, which pays off only when the number of + distinct values is small. The value it holds is exactly the ``T`` it wraps, + so two columns differ in encoding rather than in content. + + On ``DataType`` because no core concept describes it, and because there is + nothing for it to *be* a different type from — ``LowCardinality(String)`` and + ``String`` hold the same values and answer the same queries. It cannot derive + from :class:`ClickHouseStringType` (nor from any other inner type, since the + inner type is a parameter): doing so would make ``isinstance(encoding, + StringType)`` true for a type that only ever *wraps* a string. + """ name = "clickhouse_low_cardinality" @@ -415,16 +800,22 @@ def __init__(self, dialect=None, *, inner_type: DataType): super().__init__(dialect) self.inner_type = inner_type - def _type_params(self) -> tuple: - return (self.inner_type,) - + PARAMETERS = ("inner_type",) # --------------------------------------------------------------------------- # JSON type # --------------------------------------------------------------------------- -class ClickHouseJSONType(DataType): - """ClickHouse ``JSON`` — native JSON type (experimental in 26.x).""" +class ClickHouseJSONType(JsonType): + """ClickHouse ``JSON`` — a native semi-structured column type. + + Distinct from :class:`ClickHouseStringType`, which stores the same document + as opaque bytes: a ``JSON`` column knows its paths and types, so + ``JSONExtract`` reads a field without parsing the whole value, and a path that + does not exist is a query error rather than an empty result. Core's + :class:`~...types.json_.JsonType` is the concept for exactly that, so this + derives from it. + """ name = "clickhouse_json" @@ -434,7 +825,25 @@ class ClickHouseJSONType(DataType): # --------------------------------------------------------------------------- class ClickHouseAggregateFunctionType(DataType): - """ClickHouse ``AggregateFunction(name, T...)``.""" + """ClickHouse ``AggregateFunction(name, T...)`` — an intermediate state. + + Not a column's storage type: it is the partially-accumulated state an + aggregate query builds in memory and writes into + ``AggregatingMergeTree``'s columns, to be merged later. Its identity is the + *function* plus the argument types, which is why ``function_name`` and + ``arg_types`` are the parameters. + + On ``DataType`` because SQL:2016 has no type for "a value that only means + something to a particular aggregate function", and because a state that is + only meaningful to a function cannot be a subtype of any of the types it will + eventually produce — a ``sum`` state is not a ``SUM(bigint)``. + """ + + # An aggregate function's state is identified by which function it is over + # which argument types; anything else would make two different aggregates + # interchangeable. + + PARAMETERS = ("function_name", "arg_types",) name = "clickhouse_aggregate_function" @@ -444,14 +853,29 @@ class ClickHouseAggregateFunctionType(DataType): def __init__(self, dialect=None, *, function_name: str, arg_types: List[DataType]): super().__init__(dialect) self.function_name = function_name - self.arg_types = list(arg_types) - - def _type_params(self) -> tuple: - return (self.function_name, tuple(self.arg_types)) + self.arg_types = tuple(arg_types) + # A tuple from construction, not from comparison: the + # argument types are identity, and state that is only + # normalised when it is hashed is mutable in every other + # respect. + PARAMETERS = ("function_name", "arg_types",) class ClickHouseSimpleAggregateFunctionType(DataType): - """ClickHouse ``SimpleAggregateFunction(name, T...)``.""" + """ClickHouse ``SimpleAggregateFunction(name, T...)`` — a mergeable state. + + A restricted :class:`ClickHouseAggregateFunctionType`: only aggregates whose + state merges by itself (``sum``, ``min``, ``max``, ``any``…) may use it, and + then it can be stored in an ordinary ``MergeTree`` column instead of needing + ``AggregatingMergeTree``. That is a different type to ClickHouse, so it is a + different class. + + On ``DataType`` for the same reason as ``AggregateFunction``: SQL:2016 has no + type for a value that only means something to a particular aggregate + function, so there is no core concept for it to be. + """ + + PARAMETERS = ("function_name", "arg_types",) name = "clickhouse_simple_aggregate_function" @@ -461,19 +885,45 @@ class ClickHouseSimpleAggregateFunctionType(DataType): def __init__(self, dialect=None, *, function_name: str, arg_types: List[DataType]): super().__init__(dialect) self.function_name = function_name - self.arg_types = list(arg_types) - - def _type_params(self) -> tuple: - return (self.function_name, tuple(self.arg_types)) + self.arg_types = tuple(arg_types) + # A tuple from construction, not from comparison: the + # argument types are identity, and state that is only + # normalised when it is hashed is mutable in every other + # respect. + PARAMETERS = ("function_name", "arg_types",) # --------------------------------------------------------------------------- -# Spatial / Geometry types (preserved from MySQL template, CH supports via -# clickhouse-connect tuple representation) +# Spatial / Geometry types +# +# SQL:2016 has no geometric types at all, so there is nothing in core to derive +# from. The seven concrete classes do share ClickHouse's own base — which is also +# the real, separately-nameable ``GEOMETRY`` type — and each carries the same +# ``srid`` because ClickHouse gives them all the same optional coordinate +# reference identifier. # --------------------------------------------------------------------------- class ClickHouseGeometryType(DataType): - """ClickHouse ``GEOMETRY`` with optional SRID.""" + """ClickHouse ``GEOMETRY`` — a geometry of any of the seven concrete kinds. + + On ``DataType`` because SQL:2016 has no geometric types at all, so there is + nothing in core to derive from. Promoting a geometric family to core is a + design decision of its own — nine such types across five backends is the + largest candidate on the datatype-hierarchy plan's shelved list, and it is + shelved precisely so that it is not settled here by inheritance. + + ClickHouse names this type as well as using it as the base of the concrete + ones, so the inheritance below is not a grouping node invented by this + project: ``Point``, ``LineString``, ``Polygon``, ``MultiPoint``, + ``MultiLineString``, ``MultiPolygon`` and ``GeometryCollection`` are all + ``GEOMETRY`` values with a known kind, and a column declared ``GEOMETRY`` + accepts any of them. + + ``srid`` is the optional coordinate reference identifier ClickHouse writes as + ``SRID ``. It is part of the type's identity here because it is part of + the written declaration — two geometries with different SRIDs are not + interchangeable without a transform. + """ name = "clickhouse_geometry" @@ -483,58 +933,105 @@ def __init__(self, dialect=None, *, srid: Optional[int] = None): super().__init__(dialect) self.srid = srid - def _type_params(self) -> tuple: - return (self.srid,) - + PARAMETERS = ("srid",) class ClickHousePointType(ClickHouseGeometryType): - """ClickHouse ``POINT`` with optional SRID.""" + """ClickHouse ``POINT`` — a single ``(x, y)`` position. + + The smallest geometry: two coordinates and no extent. Separate from + ``GEOMETRY`` because the column will only ever hold points. + """ name = "clickhouse_point" class ClickHouseLineStringType(ClickHouseGeometryType): - """ClickHouse ``LINESTRING`` with optional SRID.""" + """ClickHouse ``LINESTRING`` — an ordered sequence of two or more positions. + + Open: it has a direction and no interior, unlike ``POLYGON``. + """ name = "clickhouse_linestring" class ClickHousePolygonType(ClickHouseGeometryType): - """ClickHouse ``POLYGON`` with optional SRID.""" + """ClickHouse ``POLYGON`` — a closed area bounded by a ring of positions. + + Has an interior, which is what separates it from ``LINESTRING``. + """ name = "clickhouse_polygon" class ClickHouseMultiPointType(ClickHouseGeometryType): - """ClickHouse ``MULTIPOINT`` with optional SRID.""" + """ClickHouse ``MULTIPOINT`` — several points held as one value. + + Not an array of points: a ``MultiPoint`` is a single geometry, which is why + it is a type rather than ``Array(Point)``. + """ name = "clickhouse_multipoint" class ClickHouseMultiLineStringType(ClickHouseGeometryType): - """ClickHouse ``MULTILINESTRING`` with optional SRID.""" + """ClickHouse ``MULTILINESTRING`` — several line strings held as one value. + + Distinct from ``Array(LINESTRING)``, which would lose the guarantee that the + collection is one geometry. + """ name = "clickhouse_multilinestring" class ClickHouseMultiPolygonType(ClickHouseGeometryType): - """ClickHouse ``MULTIPOLYGON`` with optional SRID.""" + """ClickHouse ``MULTIPOLYGON`` — several polygons held as one value. + + Distinct from ``Array(POLYGON)``; see ``MULTILINESTRING``. + """ name = "clickhouse_multipolygon" class ClickHouseGeometryCollectionType(ClickHouseGeometryType): - """ClickHouse ``GEOMETRYCOLLECTION`` with optional SRID.""" + """ClickHouse ``GEOMETRYCOLLECTION`` — geometries of mixed kinds as one value. + + The only kind that may hold any of the other six, which is why it cannot be + derived from any single one of them. + """ name = "clickhouse_geometrycollection" # --------------------------------------------------------------------------- -# VECTOR type (ClickHouse 9.0+) +# QBit — ClickHouse's vector type # --------------------------------------------------------------------------- class ClickHouseVectorType(DataType): - """ClickHouse ``VECTOR(n)`` — vector type (ClickHouse 9.0+).""" + """ClickHouse ``QBit(Float32, n)`` — a fixed-dimension vector of floats. + + ``n`` is part of the identity: a vector column cannot change its dimension + in place without rewriting the data. + + On ``DataType`` because SQL:2016 has no vector type, and because it is not an + array — the components are addressed positionally and the dimension is fixed, + where :class:`ClickHouseArrayType` allows both to vary. + + The name is historical and deliberately kept: ``VECTOR(n)`` is **MySQL 9.0's** + spelling, imported into this backend along with MySQL's ``STRING_TO_VECTOR`` / + ``VECTOR_TO_STRING`` / ``DISTANCE_*`` family, and ClickHouse has never had + it — ``CREATE TABLE t (a VECTOR(4))`` is rejected with ``Unknown data type + family: VECTOR``. ClickHouse's own vector type is ``QBit``, so that is what + this class renders. ``name`` stays ``clickhouse_vector`` because the concept + really is ClickHouse-specific rather than a core one, and the dispatch key + must carry the ``clickhouse_`` prefix; it is the *rendering* that was wrong, + not the name. + + ``QBit``'s own spelling is ``QBit(element_type, dimension[, stride])``, with + ``element_type`` one of ``Int8``, ``BFloat16``, ``Float32`` or ``Float64``. + This class carries only the dimension, so the formatter fixes the element + type at ``Float32`` — the representation this backend's own documentation + already recommends for embeddings (``Array(Float32)`` + ``L2Distance``). + """ name = "clickhouse_vector" @@ -544,5 +1041,4 @@ def __init__(self, dialect=None, *, dim: int): super().__init__(dialect) self.dim = dim - def _type_params(self) -> tuple: - return (self.dim,) + PARAMETERS = ("dim",) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/vector.py b/src/rhosocial/activerecord/backend/impl/clickhouse/expression/vector.py deleted file mode 100644 index 84530c0..0000000 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/expression/vector.py +++ /dev/null @@ -1,246 +0,0 @@ -# src/rhosocial/activerecord/backend/impl/clickhouse/expression/vector.py -""" -ClickHouse-specific vector expression functions. - -This module provides expression classes for ClickHouse vector functions: -- ClickHouseVectorLiteralExpression -- ClickHouseStringToVectorExpression -- ClickHouseVectorToStringExpression -- ClickHouseVectorDimExpression -- ClickHouseDistanceEuclideanExpression -- ClickHouseDistanceCosineExpression -- ClickHouseDistanceDotExpression -- ClickHouseCreateVectorIndexExpression - -Note: Vector support requires ClickHouse 9.0+ -""" - -from typing import TYPE_CHECKING, List, Optional - -from rhosocial.activerecord.backend.expression.bases import SQLValueExpression -from rhosocial.activerecord.backend.expression.mixins import ( - AliasableMixin, - ComparisonMixin, -) - -if TYPE_CHECKING: - from rhosocial.activerecord.backend.dialect import SQLDialectBase - - -class ClickHouseVectorLiteralExpression(AliasableMixin, SQLValueExpression): - """ClickHouse vector literal expression. - - Creates a vector value from a list of floats. - - Example: - >>> expr = ClickHouseVectorLiteralExpression(dialect, [1.0, 2.0, 3.0]) - """ - - def __init__( - self, - dialect: "SQLDialectBase", - values: List[float], - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.values = values - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_vector_literal" - - -class ClickHouseStringToVectorExpression(AliasableMixin, SQLValueExpression): - """ClickHouse STRING_TO_VECTOR expression. - - Converts a string representation to a vector. - - Example: - >>> expr = ClickHouseStringToVectorExpression(dialect, '[1,2,3]') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - vector_str: str, - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.vector_str = vector_str - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_string_to_vector" - - -class ClickHouseVectorToStringExpression(AliasableMixin, SQLValueExpression): - """ClickHouse VECTOR_TO_STRING expression. - - Converts a vector to a string representation. - - Example: - >>> expr = ClickHouseVectorToStringExpression(dialect, 'vec_col') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - vector_col: str, - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.vector_col = vector_col - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_vector_to_string" - - -class ClickHouseVectorDimExpression(AliasableMixin, SQLValueExpression): - """ClickHouse VECTOR_DIM expression. - - Returns the dimension of a vector. - - Example: - >>> expr = ClickHouseVectorDimExpression(dialect, 'vec_col') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - vector_col: str, - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.vector_col = vector_col - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_vector_dim" - - -class ClickHouseDistanceEuclideanExpression(AliasableMixin, ComparisonMixin, SQLValueExpression): - """ClickHouse Euclidean distance expression. - - Example: - >>> expr = ClickHouseDistanceEuclideanExpression(dialect, 'vec1', 'vec2') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - vec1: str, - vec2: str, - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.vec1 = vec1 - self.vec2 = vec2 - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_distance_euclidean" - - -class ClickHouseDistanceCosineExpression(AliasableMixin, ComparisonMixin, SQLValueExpression): - """ClickHouse Cosine distance expression. - - Example: - >>> expr = ClickHouseDistanceCosineExpression(dialect, 'vec1', 'vec2') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - vec1: str, - vec2: str, - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.vec1 = vec1 - self.vec2 = vec2 - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_distance_cosine" - - -class ClickHouseDistanceDotExpression(AliasableMixin, ComparisonMixin, SQLValueExpression): - """ClickHouse Dot product distance expression. - - Example: - >>> expr = ClickHouseDistanceDotExpression(dialect, 'vec1', 'vec2') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - vec1: str, - vec2: str, - *, - alias: Optional[str] = None, - ): - super().__init__(dialect) - self.vec1 = vec1 - self.vec2 = vec2 - self.alias = alias - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_distance_dot" - - -class ClickHouseCreateVectorIndexExpression(SQLValueExpression): - """ClickHouse CREATE VECTOR INDEX expression. - - Example: - >>> expr = ClickHouseCreateVectorIndexExpression(dialect, 'idx', 'table', 'column') - """ - - def __init__( - self, - dialect: "SQLDialectBase", - index_name: str, - table_name: str, - column_name: str, - ): - super().__init__(dialect) - self.index_name = index_name - self.table_name = table_name - self.column_name = column_name - - @property - def format_method(self) -> str: - """The dialect formatting method that renders this expression.""" - return "format_create_vector_index" - - -__all__ = [ - "ClickHouseVectorLiteralExpression", - "ClickHouseStringToVectorExpression", - "ClickHouseVectorToStringExpression", - "ClickHouseVectorDimExpression", - "ClickHouseDistanceEuclideanExpression", - "ClickHouseDistanceCosineExpression", - "ClickHouseDistanceDotExpression", - "ClickHouseCreateVectorIndexExpression", -] diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/introspector.py index 6f11229..d25acf0 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/introspector.py @@ -82,8 +82,15 @@ def _get_default_schema(self) -> str: return "" def _get_version(self) -> tuple: - """Return the ClickHouse server version tuple from the backend.""" - return getattr(self._backend, "_version", (8, 0, 0)) + """Return the ClickHouse server version tuple from the backend. + + Falls back to ``(0, 0, 0)``, the tuple + :meth:`...backend.backend.ClickHouseBackend.get_server_version` itself uses + for "not adapted yet", rather than to an invented version — this used to + return ``(8, 0, 0)``, which is not a release ClickHouse has and which a + caller could not distinguish from a server-reported version. + """ + return getattr(self._backend, "_version", None) or (0, 0, 0) # ------------------------------------------------------------------ # # Parse methods — pure Python, no I/O diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/show_introspector.py b/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/show_introspector.py index 3bebd03..eaa5258 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/show_introspector.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/show_introspector.py @@ -61,94 +61,164 @@ def dialect(self): @staticmethod def _parse_create_table(rows: List[Dict], table: str): + """Parse ``SHOW CREATE TABLE``. + + ClickHouse answers with a single column named ``statement`` + (verified on 26.7.3.19: ``SHOW CREATE TABLE t FORMAT JSONEachRow`` + yields ``{"statement":"CREATE TABLE default.t\\n(...)"}``), where MySQL + answers with two columns, ``Table`` and ``Create Table``. Reading the + MySQL names here would return an empty statement rather than fail, so + the ClickHouse column is read and MySQL's are not consulted. + """ from ..show.types import ShowCreateTableResult if not rows: return None row = rows[0] return ShowCreateTableResult( - table_name=row.get("Table", row.get("TABLE", table)), - create_statement=row.get("Create Table", row.get("CREATE TABLE", "")), + table_name=table, + create_statement=row.get("statement", ""), ) @staticmethod def _parse_create_view(rows: List[Dict], view_name: str): + """Parse ``SHOW CREATE VIEW``. + + Single ``statement`` column, as for ``SHOW CREATE TABLE``; the + ``character_set_client`` / ``collation_connection`` fields ClickHouse + does not report stay ``None``. + """ from ..show.types import ShowCreateViewResult if not rows: return None row = rows[0] return ShowCreateViewResult( - view_name=row.get("View", row.get("VIEW", view_name)), - create_statement=row.get("Create View", row.get("CREATE VIEW", "")), - character_set_client=row.get("character_set_client"), - collation_connection=row.get("collation_connection"), + view_name=view_name, + create_statement=row.get("statement", ""), ) @staticmethod def _parse_columns(rows: List[Dict]): + """Parse ``SHOW [FULL] COLUMNS``. + + ClickHouse's columns are lower-cased: ``field``, ``type``, ``null``, + ``key``, ``default``, ``extra``, plus ``collation``, ``comment`` and + ``privileges`` under ``FULL`` (verified on 26.7.3.19 against + ``SHOW FULL COLUMNS FROM test_db.t``). MySQL's ``Field`` / ``Type`` / + ``Null`` / ``Key`` / ``Default`` / ``Extra`` / ``Collation`` / + ``Privileges`` / ``Comment`` are read nowhere here, because reading them + yields ``None`` for every field instead of failing. + + Two ClickHouse-specific readings worth stating, both from the ``SHOW`` + reference page and both reproduced by the live rows: + + * ``key`` is ``'PRI SOR'`` for a column in the sorting key, ``'PRI'`` for + a primary-key-only column and ``''`` otherwise — not MySQL's + ``PRI``/``UNI``/``MUL``. + * ``default`` carries the expression of an ``ALIAS``, ``DEFAULT`` **or** + ``MATERIALIZED`` column (ClickHouse reports all three through this one + column), and ``extra`` is always ``''`` because the server documents + it as unused. + + The rows come back ordered by column **name**, not by declaration order: + a table declared ``(id UInt32, created_at DateTime)`` yields + ``created_at`` then ``id``. Declaration order lives in + ``SHOW CREATE TABLE`` / ``system.columns``. + """ from ..show.types import ShowColumnResult - columns = [] - for row in rows: - col = ShowColumnResult( - field=row.get("Field", row.get("COLUMN_NAME")), - type=row.get("Type", row.get("COLUMN_TYPE")), - null=row.get("Null", row.get("IS_NULLABLE")), - key=row.get("Key", row.get("COLUMN_KEY")), - default=row.get("Default", row.get("COLUMN_DEFAULT")), - extra=row.get("Extra", row.get("EXTRA")), + return [ + ShowColumnResult( + field=row.get("field"), + type=row.get("type"), + null=row.get("null"), + key=row.get("key"), + default=row.get("default"), + extra=row.get("extra"), + collation=row.get("collation"), + comment=row.get("comment"), + privileges=row.get("privileges"), ) - if "Collation" in row or "Privileges" in row: - col.privileges = row.get("Privileges") - col.comment = row.get("Comment") - columns.append(col) - return columns + for row in rows + ] @staticmethod def _parse_indexes(rows: List[Dict]): + """Parse ``SHOW INDEX``. + + ClickHouse answers with its own names, which differ from MySQL's in one + place that matters: ``pk_col`` where MySQL says ``Column_name`` + (verified on 26.7.3.19 — ``SHOW INDEX FROM test_db.t`` reports ``table``, + ``non_unique``, ``key_name``, ``seq_in_index``, ``pk_col``, ``collation``, + ``cardinality``, ``sub_part``, ``packed``, ``null``, ``index_type``, + ``comment``, ``index_comment``, ``visible``, ``expression``; the + ``SHOW`` reference page still documents that column as ``column_name``, + which the server no longer sends). + + The rows are the table's **primary key columns** (``key_name='PRIMARY'``, + ``pk_col`` holding the column name, ``collation='A'``) plus one row per + **data skipping index** (``key_name=``, ``pk_col=''`` and the + index expression in ``expression``). + """ from ..show.types import ShowIndexResult return [ ShowIndexResult( - table_name=row.get("Table", row.get("TABLE_NAME")), - non_unique=row.get("Non_unique", row.get("NON_UNIQUE")), - key_name=row.get("Key_name", row.get("INDEX_NAME")), - seq_in_index=row.get("Seq_in_index", row.get("SEQ_IN_INDEX")), - column_name=row.get("Column_name", row.get("COLUMN_NAME")), - collation=row.get("Collation", row.get("COLLATION")), - cardinality=row.get("Cardinality", row.get("CARDINALITY")), - sub_part=row.get("Sub_part", row.get("SUB_PART")), - packed=row.get("Packed", row.get("PACKED")), - null=row.get("Null", row.get("NULLABLE")), - index_type=row.get("Index_type", row.get("INDEX_TYPE"), "BTREE"), - comment=row.get("Comment", row.get("INDEX_COMMENT")), - index_comment=row.get("Index_comment", row.get("INDEX_COMMENT")), - visible=row.get("Visible", row.get("IS_VISIBLE")), - expression=row.get("Expression", row.get("EXPRESSION")), + table_name=row.get("table"), + non_unique=row.get("non_unique"), + key_name=row.get("key_name"), + seq_in_index=row.get("seq_in_index"), + column_name=row.get("pk_col"), + collation=row.get("collation"), + cardinality=row.get("cardinality"), + sub_part=row.get("sub_part"), + packed=row.get("packed"), + null=row.get("null"), + index_type=row.get("index_type"), + comment=row.get("comment"), + index_comment=row.get("index_comment"), + visible=row.get("visible"), + expression=row.get("expression"), ) for row in rows ] @staticmethod def _parse_tables(rows: List[Dict]): + """Parse ``SHOW [FULL] TABLES``. + + ClickHouse names the column ``name`` in both modes and adds an + ``engine`` column under ``FULL`` — verified on 26.7.3.19, where + ``SHOW TABLES LIMIT 1 FORMAT JSONCompact`` reports + ``[{"name": "String"}]`` and the ``FULL`` form adds + ``{"name": "String", "engine": "String"}``. MySQL instead names the + first column after the database (``Tables_in_``) and calls the + second ``Table_type``; reading those names here dropped every ``FULL`` + row, so the ClickHouse names are used. + + ``ShowTableResult.table_type`` therefore carries the **storage engine** + name (``MergeTree``, ``ReplicatedMergeTree``, ``View``, ...) on this + backend, which is what ClickHouse reports there, not MySQL's + ``BASE TABLE`` / ``VIEW``. + """ from ..show.types import ShowTableResult - result = [] - for row in rows: - if len(row) == 1: - result.append(ShowTableResult(name=list(row.values())[0], table_type=None)) - else: - name_key = next((k for k in row.keys() if k.startswith("Tables_in_")), None) - if name_key: - result.append(ShowTableResult(name=row[name_key], table_type=row.get("Table_type"))) - return result + return [ + ShowTableResult(name=row.get("name"), table_type=row.get("engine")) + for row in rows + ] @staticmethod def _parse_databases(rows: List[Dict]): + """Parse ``SHOW DATABASES``. + + The column is ``name`` (verified on 26.7.3.19); MySQL calls it + ``Database``. + """ from ..show.types import ShowDatabaseResult - return [ShowDatabaseResult(name=row.get("Database")) for row in rows] + return [ShowDatabaseResult(name=row.get("name")) for row in rows] @staticmethod def _parse_table_status(rows: List[Dict]): @@ -228,21 +298,58 @@ def _parse_status(rows: List[Dict]): @staticmethod def _parse_processlist(rows: List[Dict]): + """Parse ``SHOW PROCESSLIST``. + + ClickHouse's ``SHOW PROCESSLIST`` is ``system.processes`` verbatim, so the + columns are that table's 43 lower-cased names (``query_id``, ``user``, + ``address``, ``port``, ``elapsed``, ``current_database``, ``query``, + ...) rather than MySQL's ``Id`` / ``User`` / ``Host`` / ``Command`` / + ``Time`` / ``db`` / ``State`` / ``Info``. + + Mapping, and the three fields with no ClickHouse counterpart: + + ==================================== ================================= + ``ShowProcessListResult`` ClickHouse column + ==================================== ================================= + ``id`` ``query_id`` (a UUID string, not + MySQL's integer thread id) + ``user`` ``user`` + ``host`` ``address`` + ``port`` joined, + as MySQL's ``Host`` does + ``command`` **none** — every row in + ``system.processes`` *is* a + running query; stays ``None`` + ``time`` ``elapsed`` (Float64 seconds) + ``db`` ``current_database`` + ``state`` **none** — no per-process state + column exists; stays ``None`` + ``info`` ``query`` + ==================================== ================================= + """ from ..show.types import ShowProcessListResult - return [ - ShowProcessListResult( - id=row.get("Id", row.get("ID")), - user=row.get("User"), - host=row.get("Host"), - command=row.get("Command"), - time=row.get("Time"), - db=row.get("db"), - state=row.get("State"), - info=row.get("Info"), + results = [] + for row in rows: + address = row.get("address") + port = row.get("port") + host = None + if address is not None and port is not None: + host = f"{address}:{port}" + elif address is not None: + host = str(address) + results.append( + ShowProcessListResult( + id=row.get("query_id"), + user=row.get("user"), + host=host, + command=None, + time=row.get("elapsed"), + db=row.get("current_database"), + state=None, + info=row.get("query"), + ) ) - for row in rows - ] + return results @staticmethod def _parse_warnings(rows: List[Dict]): @@ -254,15 +361,49 @@ def _parse_warnings(rows: List[Dict]): @staticmethod def _parse_engines(rows: List[Dict]): + """Parse ``SHOW ENGINES``. + + ClickHouse's columns are its own, read here as the server sends them: + ``name``, the eight ``supports_*`` flags (``supports_settings``, + ``supports_skipping_indices``, ``supports_projections``, + ``supports_sort_order``, ``supports_ttl``, ``supports_replication``, + ``supports_deduplication``, ``supports_parallel_insert`` — each ``UInt8`` + 0/1), then ``description``, ``syntax``, ``examples``, ``introduced_in`` + and ``related`` — the last five only where the server has them. + + Measured: 25.8.33.6 and 26.3.28.5 carry ``name`` plus the eight flags + and nothing else, and a literal ``SELECT syntax FROM system.table_engines`` + on 25.8 is refused with ``UNKNOWN_IDENTIFIER``; 26.7.3.19 and 26.7.21.2 + carry all fourteen. (Which release added them was not determined — only + the 25.8/26.3 versus 26.7 boundary was.) The older versions therefore + yield ``None`` for ``description`` / ``syntax`` / ``examples`` / + ``introduced_in`` / ``related``, which is the only honest answer — the + statement does not produce them, so no value may be invented for them. + + ``introduced_in`` is the only version-shaped column in the whole ``SHOW`` + set, and 26.7 leaves it empty: ``SELECT count() FROM + system.table_engines WHERE introduced_in != ''`` returns 0 across all 83 + engines on 26.7.3.19. MySQL's ``Support`` / ``Transactions`` / ``XA`` / + ``Savepoints`` are not produced by this statement and are not read. + """ from ..show.types import ShowEngineResult return [ ShowEngineResult( - engine=row.get("Engine"), - support=row.get("Support"), - transactions=row.get("Transactions"), - xa=row.get("XA"), - savepoints=row.get("Savepoints"), + engine=row.get("name"), + supports_settings=row.get("supports_settings"), + supports_skipping_indices=row.get("supports_skipping_indices"), + supports_projections=row.get("supports_projections"), + supports_sort_order=row.get("supports_sort_order"), + supports_ttl=row.get("supports_ttl"), + supports_replication=row.get("supports_replication"), + supports_deduplication=row.get("supports_deduplication"), + supports_parallel_insert=row.get("supports_parallel_insert"), + description=row.get("description"), + syntax=row.get("syntax"), + examples=row.get("examples"), + introduced_in=row.get("introduced_in"), + related=row.get("related"), ) for row in rows ] @@ -299,9 +440,37 @@ def _parse_collation(rows: List[Dict]): @staticmethod def _parse_grants(rows: List[Dict]): + """Parse ``SHOW GRANTS``. + + The statement returns **one column whose name is not a name at all**: + ClickHouse labels it after the statement text plus whatever output format + the client asked for. Measured on 26.7.3.19: + + * ``SHOW GRANTS`` over HTTP with ``default_format=JSONEachRow`` → + column ``GRANTS`` + * ``SHOW GRANTS FORMAT JSONEachRow`` → column ``GRANTS FORMAT JSONEachRow`` + * ``SHOW GRANTS`` over the native protocol (clickhouse-connect, which + appends ``FORMAT Native``) → column ``GRANTS FORMAT Native`` + * ``SHOW GRANTS FOR root`` → column ``GRANTS FOR root`` + + So there is no fixed identifier to look for, and MySQL's ``Grants for`` + never appears. The statement has exactly one column, so the single value + in each row is read: first by the ``GRANTS``-prefix rule the server's own + naming obeys, then positionally if even that does not match. + """ from ..show.types import ShowGrantResult - return [ShowGrantResult(grants=row.get("Grants for")) for row in rows] + results = [] + for row in rows: + grants = None + grants_key = next((k for k in row if k.upper().startswith("GRANTS")), None) + if grants_key is not None: + grants = row[grants_key] + elif len(row) == 1: + # Exactly one column, whatever it is called: take its value. + grants = next(iter(row.values())) + results.append(ShowGrantResult(grants=grants)) + return results @staticmethod def _parse_plugins(rows: List[Dict]): diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/status_introspector.py b/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/status_introspector.py index 7037531..1c1b288 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/status_introspector.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/introspection/status_introspector.py @@ -2,8 +2,59 @@ """ ClickHouse server status introspector. -Provides server status information by querying ClickHouse's SHOW VARIABLES, -SHOW STATUS, and other system commands. +Every value reported here is read out of a ClickHouse ``system.*`` table. +ClickHouse has **no** ``SHOW VARIABLES`` and **no** ``SHOW STATUS``: on +26.7.3.19 both are rejected by the parser with ``SYNTAX_ERROR ... Expected one +of: CREATE, FULL, DATABASES, CLUSTERS, MERGES, ...``, so the two inventories +below name rows that were read off the server rather than MySQL variables that +were hoped for. + +The five inventories are **requests, not claims**. Each is a shortlist of names +worth reporting plus the category each is filed under; what the caller receives +is the intersection of that shortlist with what the connected server actually +holds. A name the server does not have is dropped rather than reported from +memory, so no inventory can ever assert a setting or metric the server lacks. + +That is not defensive padding — a fixed name list asserting server contents is +simply false across versions. Measured on this machine, the row counts are: + +================= ========================= ====== ====== ====== +constant source table 25.8 26.3 26.7 +================= ========================= ====== ====== ====== +CONFIG_SETTINGS ``system.settings`` 1370 1550 1715 +SERVER_SETTINGS ``system.server_settings`` 228 381 439 +GAUGE_METRICS ``system.metrics`` 417 469 521 +COUNTER_EVENTS ``system.events`` 144* 157* 225* +ASYNC_METRICS ``system.asynchronous_metrics`` 910 921 944 +================= ========================= ====== ====== ====== + +(*) ``system.events`` is not a fixed catalogue at all: a row appears only once +that event has actually fired, so the count depends on the server's *history*, +not on its version. A freshly started 26.7.21.2 reports 115 event rows and is +missing 20 of the 49 names in :data:`CLICKHOUSE_COUNTER_EVENTS`; the same +container reaches 171 rows once a merge, an insert, a mutation and a failed +query have run. This is why the inventories are requests: there is no version +key under which a fixed event list could be a true statement. + +``system.asynchronous_metrics`` likewise grows with the CPU / disk / +network-interface count, which is why that inventory is restricted to names +carrying no device suffix. + +Only two things in an entry are this backend's judgement: the **name** and the +:class:`StatusCategory` it is filed under. Descriptions, defaults, units and +read-only flags are all read from the row the server returned — +``system.settings`` and ``system.server_settings`` expose ``description``, +``default``, ``readonly`` / ``changeable_without_restart`` and ``type``, and the +three metric tables expose ``description`` — so nothing here restates the server +in prose that can drift from it. Each item also records its originating table in +``StatusItem.extra['source']``. + +The category rule: a setting or metric is filed under ``STORAGE`` if it is about +files, disks, parts or compressed bytes, under ``CONNECTION`` if it is about +connections, sessions or concurrency admission, under ``SECURITY`` if it is about +access or credentials, under ``REPLICATION`` if it is about replicas, quorum or +consistency, under ``PERFORMANCE`` if it is a throughput limit or switch, and +under ``CONFIGURATION`` otherwise. Design principle: Sync and Async are separate and cannot coexist. - SyncClickHouseStatusIntrospector: for synchronous backends @@ -11,7 +62,7 @@ """ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple from rhosocial.activerecord.backend.introspection.status import ( StatusItem, @@ -22,119 +73,326 @@ ConnectionInfo, StorageInfo, SessionInfo, - InnoDBInfo, - BinaryLogInfo, ProcessInfo, - SlowQueryInfo, SyncAbstractStatusIntrospector, AsyncAbstractStatusIntrospector, ) -# ClickHouse variables to include in status overview -# Format: (variable_name, category, description, unit, is_readonly) -CLICKHOUSE_CONFIG_VARIABLES = [ - # Configuration - ("version", StatusCategory.CONFIGURATION, "ClickHouse server version", None, True), - ("version_comment", StatusCategory.CONFIGURATION, "Version comment", None, True), - ("version_compile_machine", StatusCategory.CONFIGURATION, "Compile machine type", None, True), - ("version_compile_os", StatusCategory.CONFIGURATION, "Compile OS", None, True), - ("port", StatusCategory.CONFIGURATION, "ClickHouse port", None, True), - ("socket", StatusCategory.CONFIGURATION, "ClickHouse socket", None, True), - ("datadir", StatusCategory.STORAGE, "Data directory", None, True), - ("basedir", StatusCategory.CONFIGURATION, "Base directory", None, True), - ("tmpdir", StatusCategory.CONFIGURATION, "Temporary directory", None, True), - ("character_set_server", StatusCategory.CONFIGURATION, "Server character set", None, False), - ("character_set_database", StatusCategory.CONFIGURATION, "Database character set", None, False), - ("character_set_client", StatusCategory.CONFIGURATION, "Client character set", None, False), - ("character_set_connection", StatusCategory.CONFIGURATION, "Connection character set", None, False), - ("character_set_results", StatusCategory.CONFIGURATION, "Results character set", None, False), - ("collation_server", StatusCategory.CONFIGURATION, "Server collation", None, False), - ("collation_database", StatusCategory.CONFIGURATION, "Database collation", None, False), - ("max_connections", StatusCategory.CONNECTION, "Maximum connections", "connections", False), - ("max_connect_errors", StatusCategory.SECURITY, "Max connect errors", None, False), - ("max_user_connections", StatusCategory.CONNECTION, "Max user connections", "connections", False), - ("connect_timeout", StatusCategory.CONFIGURATION, "Connection timeout", "seconds", False), - ("wait_timeout", StatusCategory.CONFIGURATION, "Wait timeout", "seconds", False), - ("interactive_timeout", StatusCategory.CONFIGURATION, "Interactive timeout", "seconds", False), - ("skip_networking", StatusCategory.SECURITY, "Skip networking", None, True), - ("skip_name_resolve", StatusCategory.CONFIGURATION, "Skip name resolution", None, False), - # Performance - ("innodb_buffer_pool_size", StatusCategory.PERFORMANCE, "InnoDB buffer pool size", "bytes", False), - ("innodb_buffer_pool_instances", StatusCategory.PERFORMANCE, "Buffer pool instances", None, False), - ("innodb_log_file_size", StatusCategory.PERFORMANCE, "InnoDB log file size", "bytes", True), - ("innodb_log_buffer_size", StatusCategory.PERFORMANCE, "InnoDB log buffer size", "bytes", False), - ("innodb_flush_log_at_trx_commit", StatusCategory.PERFORMANCE, "Flush log at trx commit", None, False), - ("innodb_lock_wait_timeout", StatusCategory.PERFORMANCE, "Lock wait timeout", "seconds", False), - ("innodb_read_io_threads", StatusCategory.PERFORMANCE, "Read I/O threads", None, True), - ("innodb_write_io_threads", StatusCategory.PERFORMANCE, "Write I/O threads", None, True), - ("key_buffer_size", StatusCategory.PERFORMANCE, "Key buffer size", "bytes", False), - ("query_cache_size", StatusCategory.PERFORMANCE, "Query cache size", "bytes", False), - ("query_cache_type", StatusCategory.PERFORMANCE, "Query cache type", None, False), - ("table_open_cache", StatusCategory.PERFORMANCE, "Table open cache", None, False), - ("thread_cache_size", StatusCategory.PERFORMANCE, "Thread cache size", None, False), - ("sort_buffer_size", StatusCategory.PERFORMANCE, "Sort buffer size", "bytes", False), - ("join_buffer_size", StatusCategory.PERFORMANCE, "Join buffer size", "bytes", False), - ("read_buffer_size", StatusCategory.PERFORMANCE, "Read buffer size", "bytes", False), - ("read_rnd_buffer_size", StatusCategory.PERFORMANCE, "Random read buffer size", "bytes", False), - # Security - ("sql_mode", StatusCategory.CONFIGURATION, "SQL mode", None, False), - ("secure_file_priv", StatusCategory.SECURITY, "Secure file privilege", None, True), - ("local_infile", StatusCategory.SECURITY, "Local infile", None, False), - # Replication - ("server_id", StatusCategory.REPLICATION, "Server ID", None, False), - ("log_bin", StatusCategory.REPLICATION, "Binary logging", None, True), - ("binlog_format", StatusCategory.REPLICATION, "Binlog format", None, False), - ("gtid_mode", StatusCategory.REPLICATION, "GTID mode", None, False), - ("read_only", StatusCategory.REPLICATION, "Read only mode", None, False), - ("super_read_only", StatusCategory.REPLICATION, "Super read only", None, False), -] - -# ClickHouse status variables for performance metrics -# Format: (variable_name, category, description, unit) -CLICKHOUSE_STATUS_VARIABLES = [ - # Connection metrics - ("Threads_connected", StatusCategory.CONNECTION, "Current connections", "connections"), - ("Threads_running", StatusCategory.CONNECTION, "Running threads", "threads"), - ("Threads_cached", StatusCategory.CONNECTION, "Cached threads", "threads"), - ("Max_used_connections", StatusCategory.CONNECTION, "Max used connections", "connections"), - ("Aborted_connects", StatusCategory.CONNECTION, "Aborted connects", None), - ("Aborted_clients", StatusCategory.CONNECTION, "Aborted clients", None), - ("Connections", StatusCategory.CONNECTION, "Total connections", "connections"), - # Performance metrics - ("Queries", StatusCategory.PERFORMANCE, "Total queries", "queries"), - ("Questions", StatusCategory.PERFORMANCE, "Total questions", None), - ("Slow_queries", StatusCategory.PERFORMANCE, "Slow queries", "queries"), - ("Qcache_hits", StatusCategory.PERFORMANCE, "Query cache hits", None), - ("Qcache_inserts", StatusCategory.PERFORMANCE, "Query cache inserts", None), - ("Qcache_lowmem_prunes", StatusCategory.PERFORMANCE, "Query cache lowmem prunes", None), - ("Com_select", StatusCategory.PERFORMANCE, "SELECT statements", None), - ("Com_insert", StatusCategory.PERFORMANCE, "INSERT statements", None), - ("Com_update", StatusCategory.PERFORMANCE, "UPDATE statements", None), - ("Com_delete", StatusCategory.PERFORMANCE, "DELETE statements", None), - ("Com_replace", StatusCategory.PERFORMANCE, "REPLACE statements", None), - ("Com_load", StatusCategory.PERFORMANCE, "LOAD DATA statements", None), - ("Bytes_received", StatusCategory.PERFORMANCE, "Bytes received", "bytes"), - ("Bytes_sent", StatusCategory.PERFORMANCE, "Bytes sent", "bytes"), - # InnoDB metrics - ("Innodb_buffer_pool_read_requests", StatusCategory.PERFORMANCE, "Buffer pool read requests", None), - ("Innodb_buffer_pool_reads", StatusCategory.PERFORMANCE, "Buffer pool reads", None), - ("Innodb_buffer_pool_wait_free", StatusCategory.PERFORMANCE, "Buffer pool wait free", None), - ("Innodb_data_reads", StatusCategory.PERFORMANCE, "Data reads", None), - ("Innodb_data_writes", StatusCategory.PERFORMANCE, "Data writes", None), - ("Innodb_data_read", StatusCategory.PERFORMANCE, "Data read", "bytes"), - ("Innodb_data_written", StatusCategory.PERFORMANCE, "Data written", "bytes"), - ("Innodb_row_lock_waits", StatusCategory.PERFORMANCE, "Row lock waits", None), - ("Innodb_row_lock_time", StatusCategory.PERFORMANCE, "Row lock time", "ms"), - ("Innodb_rows_read", StatusCategory.PERFORMANCE, "Rows read", "rows"), - ("Innodb_rows_inserted", StatusCategory.PERFORMANCE, "Rows inserted", "rows"), - ("Innodb_rows_updated", StatusCategory.PERFORMANCE, "Rows updated", "rows"), - ("Innodb_rows_deleted", StatusCategory.PERFORMANCE, "Rows deleted", "rows"), -] +# --------------------------------------------------------------------------- # +# Inventories — (name, StatusCategory). +# +# Each entry is a request for a name plus this backend's judgement about the +# category it belongs to. It is not a claim that the name exists: every list +# below was drawn from a 26.7.3.19 server, and the five tables differ between +# releases (see the module docstring for the measured counts). The assembly +# methods drop any name the connected server does not return, so the caller +# gets the server's answer and never this module's guess. +# +# ``tests/.../introspection/test_status_introspector.py`` pins the resulting +# contract from a live connection on every version: nothing is returned that +# the server does not have, nothing is dropped that it does, and no MySQL name +# survives in any list. +# --------------------------------------------------------------------------- # + +# ``system.settings`` — settings a query or the session can change at runtime. +CLICKHOUSE_CONFIG_SETTINGS: Tuple[Tuple[str, StatusCategory], ...] = ( + # execution shape + ("max_threads", StatusCategory.PERFORMANCE), + ("max_block_size", StatusCategory.PERFORMANCE), + ("max_insert_block_size", StatusCategory.PERFORMANCE), + ("preferred_block_size_bytes", StatusCategory.PERFORMANCE), + ("max_memory_usage", StatusCategory.PERFORMANCE), + ("max_execution_time", StatusCategory.PERFORMANCE), + ("max_result_rows", StatusCategory.PERFORMANCE), + ("max_result_bytes", StatusCategory.PERFORMANCE), + ("max_rows_to_read", StatusCategory.PERFORMANCE), + ("max_columns_to_read", StatusCategory.PERFORMANCE), + ("max_parser_depth", StatusCategory.CONFIGURATION), + ("short_circuit_function_evaluation", StatusCategory.CONFIGURATION), + ("join_algorithm", StatusCategory.PERFORMANCE), + ("join_use_nulls", StatusCategory.CONFIGURATION), + ("optimize_move_to_prewhere", StatusCategory.PERFORMANCE), + ("group_by_two_level_threshold", StatusCategory.PERFORMANCE), + ("prefer_column_name_to_alias", StatusCategory.CONFIGURATION), + ("transform_null_in", StatusCategory.CONFIGURATION), + ("validate_enum_literals_in_operators", StatusCategory.CONFIGURATION), + ("normalize_function_names", StatusCategory.CONFIGURATION), + ("print_pretty_type_names", StatusCategory.CONFIGURATION), + ("use_client_time_zone", StatusCategory.CONFIGURATION), + ("session_timezone", StatusCategory.CONFIGURATION), + ("max_download_threads", StatusCategory.PERFORMANCE), + ("distributed_foreground_insert", StatusCategory.PERFORMANCE), + # concurrency admission + ("max_concurrent_queries_for_user", StatusCategory.CONNECTION), + ("max_concurrent_queries_for_all_users", StatusCategory.CONNECTION), + # caches + ("use_query_cache", StatusCategory.PERFORMANCE), + ("query_cache_max_size_in_bytes", StatusCategory.PERFORMANCE), + ("query_cache_ttl", StatusCategory.PERFORMANCE), + ("use_uncompressed_cache", StatusCategory.PERFORMANCE), + ("max_bytes_before_external_sort", StatusCategory.PERFORMANCE), + ("max_bytes_before_external_group_by", StatusCategory.PERFORMANCE), + ("max_bytes_before_external_join", StatusCategory.PERFORMANCE), + # files and compression + ("max_compress_block_size", StatusCategory.STORAGE), + ("min_compress_block_size", StatusCategory.STORAGE), + ("local_filesystem_read_method", StatusCategory.STORAGE), + ("storage_file_read_method", StatusCategory.STORAGE), + ("min_free_disk_bytes_to_perform_insert", StatusCategory.STORAGE), + ("max_partition_size_to_drop", StatusCategory.STORAGE), + ("max_table_size_to_drop", StatusCategory.STORAGE), + # connections + ("http_connection_timeout", StatusCategory.CONNECTION), + ("http_receive_timeout", StatusCategory.CONNECTION), + ("http_send_timeout", StatusCategory.CONNECTION), + ("receive_timeout", StatusCategory.CONNECTION), + ("send_timeout", StatusCategory.CONNECTION), + ("tcp_keep_alive_timeout", StatusCategory.CONNECTION), + ("idle_connection_timeout", StatusCategory.CONNECTION), + ("max_distributed_connections", StatusCategory.CONNECTION), + # access + ("readonly", StatusCategory.SECURITY), + ("s3_allow_server_credentials_in_user_queries", StatusCategory.SECURITY), + # replicas + ("load_balancing", StatusCategory.REPLICATION), + ("parallel_replicas_mode", StatusCategory.REPLICATION), + ("prefer_localhost_replica", StatusCategory.REPLICATION), + ("insert_quorum", StatusCategory.REPLICATION), + ("insert_quorum_parallel", StatusCategory.REPLICATION), + ("insert_quorum_timeout", StatusCategory.REPLICATION), + ("replication_alter_partitions_sync", StatusCategory.REPLICATION), + ("replication_wait_for_inactive_replica_timeout", StatusCategory.REPLICATION), + ("select_sequential_consistency", StatusCategory.REPLICATION), + ("update_sequential_consistency", StatusCategory.REPLICATION), +) + +# ``system.server_settings`` — the ``config.xml`` half of the configuration. +# ``max_connections`` lives here and *only* here: it is not a +# ``system.settings`` row, so a ``system.settings`` lookup for it returns +# nothing at all. +CLICKHOUSE_SERVER_SETTINGS: Tuple[Tuple[str, StatusCategory], ...] = ( + ("max_connections", StatusCategory.CONNECTION), + ("max_concurrent_queries", StatusCategory.CONNECTION), + ("listen_backlog", StatusCategory.CONNECTION), + ("keep_alive_timeout", StatusCategory.CONNECTION), + ("max_thread_pool_size", StatusCategory.PERFORMANCE), + ("max_server_memory_usage", StatusCategory.PERFORMANCE), + ("mark_cache_size", StatusCategory.PERFORMANCE), + ("uncompressed_cache_size", StatusCategory.PERFORMANCE), + ("primary_index_cache_size", StatusCategory.PERFORMANCE), + ("page_cache_size_ratio", StatusCategory.STORAGE), + ("max_held_snapshots", StatusCategory.STORAGE), + ("path", StatusCategory.STORAGE), + ("tmp_path", StatusCategory.STORAGE), +) + +# ``system.metrics`` — instantaneous gauges, reset on restart. +CLICKHOUSE_GAUGE_METRICS: Tuple[Tuple[str, StatusCategory], ...] = ( + # running work + ("Query", StatusCategory.PERFORMANCE), + ("QueryNonInternal", StatusCategory.PERFORMANCE), + ("Merge", StatusCategory.PERFORMANCE), + ("PartMutation", StatusCategory.PERFORMANCE), + ("BackgroundMergesAndMutationsPoolTask", StatusCategory.PERFORMANCE), + ("Compressing", StatusCategory.PERFORMANCE), + ("Decompressing", StatusCategory.PERFORMANCE), + ("Read", StatusCategory.PERFORMANCE), + ("Write", StatusCategory.PERFORMANCE), + ("RemoteRead", StatusCategory.PERFORMANCE), + ("DistributedSend", StatusCategory.PERFORMANCE), + ("ConcurrentQueryAcquired", StatusCategory.CONNECTION), + ("ConcurrentQueryScheduled", StatusCategory.CONNECTION), + ("RWLockActiveReaders", StatusCategory.CONNECTION), + ("RWLockActiveWriters", StatusCategory.CONNECTION), + # memory + ("MemoryTracking", StatusCategory.PERFORMANCE), + ("MemoryTrackingUncorrected", StatusCategory.PERFORMANCE), + ("MemoryReservationDemand", StatusCategory.PERFORMANCE), + ("MemoryReservationApproved", StatusCategory.PERFORMANCE), + # caches and query pipeline + ("QueryCacheBytes", StatusCategory.PERFORMANCE), + ("QueryCacheEntries", StatusCategory.PERFORMANCE), + ("AsynchronousInsertQueueSize", StatusCategory.PERFORMANCE), + ("AsynchronousInsertQueueBytes", StatusCategory.PERFORMANCE), + ("PendingAsyncInsert", StatusCategory.PERFORMANCE), + ("DelayedInserts", StatusCategory.PERFORMANCE), + ("BrokenDistributedBytesToInsert", StatusCategory.PERFORMANCE), + # parts and files on disk + ("MergeParts", StatusCategory.STORAGE), + ("PartsOutdated", StatusCategory.STORAGE), + ("PartsWide", StatusCategory.STORAGE), + ("PartsTemporary", StatusCategory.STORAGE), + ("TablesToDropQueueSize", StatusCategory.STORAGE), + ("DiskSpaceReservedForMerge", StatusCategory.STORAGE), + ("TemporaryFilesForSort", StatusCategory.STORAGE), + ("TotalTemporaryFiles", StatusCategory.STORAGE), + ("FilesystemCacheSize", StatusCategory.STORAGE), + ("AttachedTable", StatusCategory.STORAGE), + ("AttachedDatabase", StatusCategory.STORAGE), + ("AttachedView", StatusCategory.STORAGE), + ("AttachedDictionary", StatusCategory.STORAGE), + ("AttachedReplicatedTable", StatusCategory.STORAGE), + # connections + ("HTTPConnection", StatusCategory.CONNECTION), + ("HTTPConnectionsStored", StatusCategory.CONNECTION), + ("HTTPConnectionsTotal", StatusCategory.CONNECTION), + ("TCPConnection", StatusCategory.CONNECTION), + ("MySQLConnection", StatusCategory.CONNECTION), + ("PostgreSQLConnection", StatusCategory.CONNECTION), + # replicas and ZooKeeper / Keeper + ("InterserverConnection", StatusCategory.CONNECTION), + ("ReplicatedFetch", StatusCategory.REPLICATION), + ("ReplicatedSend", StatusCategory.REPLICATION), + ("ReplicatedChecks", StatusCategory.REPLICATION), + ("ReplicaReady", StatusCategory.REPLICATION), + ("ReadonlyReplica", StatusCategory.REPLICATION), + ("SharedMergeTreeFetch", StatusCategory.REPLICATION), + ("SharedMergeTreeMaxPartitions", StatusCategory.REPLICATION), + ("ZooKeeperSession", StatusCategory.REPLICATION), + ("ZooKeeperWatch", StatusCategory.REPLICATION), + ("ZooKeeperConnectionLossStartedTimestampSeconds", StatusCategory.REPLICATION), + ("KeeperAliveConnections", StatusCategory.REPLICATION), + # server identity / state + ("Revision", StatusCategory.CONFIGURATION), + ("VersionInteger", StatusCategory.CONFIGURATION), + ("IsServerShuttingDown", StatusCategory.CONFIGURATION), + ("LicenseRemainingSeconds", StatusCategory.CONFIGURATION), +) + +# ``system.events`` — monotonic counters since server start. +CLICKHOUSE_COUNTER_EVENTS: Tuple[Tuple[str, StatusCategory], ...] = ( + ("Query", StatusCategory.PERFORMANCE), + ("InitialQuery", StatusCategory.PERFORMANCE), + ("SelectQuery", StatusCategory.PERFORMANCE), + ("InitialSelectQuery", StatusCategory.PERFORMANCE), + ("InsertQuery", StatusCategory.PERFORMANCE), + ("FailedQuery", StatusCategory.PERFORMANCE), + ("FailedSelectQuery", StatusCategory.PERFORMANCE), + ("FailedInsertQuery", StatusCategory.PERFORMANCE), + ("QueryTimeMicroseconds", StatusCategory.PERFORMANCE), + ("SelectQueryTimeMicroseconds", StatusCategory.PERFORMANCE), + ("InsertQueryTimeMicroseconds", StatusCategory.PERFORMANCE), + ("RealTimeMicroseconds", StatusCategory.PERFORMANCE), + ("UserTimeMicroseconds", StatusCategory.PERFORMANCE), + ("SystemTimeMicroseconds", StatusCategory.PERFORMANCE), + ("QueryParseMicroseconds", StatusCategory.PERFORMANCE), + ("QueryAnalysisMicroseconds", StatusCategory.PERFORMANCE), + ("QueryPlanOptimizeMicroseconds", StatusCategory.PERFORMANCE), + ("QueryPipelineBuildMicroseconds", StatusCategory.PERFORMANCE), + ("SelectedRows", StatusCategory.PERFORMANCE), + ("SelectedBytes", StatusCategory.PERFORMANCE), + ("SelectedMarks", StatusCategory.PERFORMANCE), + ("SelectedParts", StatusCategory.PERFORMANCE), + ("SelectQueriesWithPrimaryKeyUsage", StatusCategory.PERFORMANCE), + ("InsertedRows", StatusCategory.PERFORMANCE), + ("InsertedBytes", StatusCategory.PERFORMANCE), + ("MergedRows", StatusCategory.PERFORMANCE), + ("MergedUncompressedBytes", StatusCategory.PERFORMANCE), + ("Merge", StatusCategory.PERFORMANCE), + ("MutatedRows", StatusCategory.PERFORMANCE), + ("MarkCacheHits", StatusCategory.PERFORMANCE), + ("MarkCacheMisses", StatusCategory.PERFORMANCE), + ("QueryConditionCacheMisses", StatusCategory.PERFORMANCE), + ("Seek", StatusCategory.PERFORMANCE), + ("FunctionExecute", StatusCategory.PERFORMANCE), + ("CompiledFunctionExecute", StatusCategory.PERFORMANCE), + ("ReadCompressedBytes", StatusCategory.PERFORMANCE), + ("RowsReadByMainReader", StatusCategory.PERFORMANCE), + ("RowsReadByPrewhereReaders", StatusCategory.PERFORMANCE), + ("NetworkReceiveBytes", StatusCategory.PERFORMANCE), + ("NetworkSendBytes", StatusCategory.PERFORMANCE), + ("ExternalProcessingFilesTotal", StatusCategory.PERFORMANCE), + ("MergeTreeDataWriterRows", StatusCategory.STORAGE), + ("MergeTreeDataWriterCompressedBytes", StatusCategory.STORAGE), + ("LoadedDataParts", StatusCategory.STORAGE), + ("LoadedMarksFiles", StatusCategory.STORAGE), + ("OSReadBytes", StatusCategory.STORAGE), + ("OSWriteBytes", StatusCategory.STORAGE), + ("FileOpen", StatusCategory.STORAGE), + ("FileSync", StatusCategory.STORAGE), +) + +# ``system.asynchronous_metrics`` — periodically sampled OS / disk / server +# figures. Restricted to names carrying no device, CPU or interface suffix, +# because those are generated per host (e.g. ``BlockReadBytes_nbd0``, +# ``OSUserTimeCPU7``) and would not be stable names. +CLICKHOUSE_ASYNC_METRICS: Tuple[Tuple[str, StatusCategory], ...] = ( + ("Uptime", StatusCategory.CONFIGURATION), + ("OSUptime", StatusCategory.CONFIGURATION), + ("AsynchronousMetricsCalculationTimeSpent", StatusCategory.CONFIGURATION), + ("LoadAverage1", StatusCategory.PERFORMANCE), + ("LoadAverage5", StatusCategory.PERFORMANCE), + ("LoadAverage15", StatusCategory.PERFORMANCE), + ("Jitter", StatusCategory.PERFORMANCE), + ("OSOpenFiles", StatusCategory.PERFORMANCE), + ("OSThreadsTotal", StatusCategory.PERFORMANCE), + ("MemoryResident", StatusCategory.PERFORMANCE), + ("MemoryVirtual", StatusCategory.PERFORMANCE), + ("TrackedMemory", StatusCategory.PERFORMANCE), + ("UntrackedMemory", StatusCategory.PERFORMANCE), + ("CGroupMemoryTotal", StatusCategory.PERFORMANCE), + ("CGroupMemoryUsed", StatusCategory.PERFORMANCE), + ("OSMemoryTotal", StatusCategory.PERFORMANCE), + ("OSMemoryAvailable", StatusCategory.PERFORMANCE), + ("QueriesMemoryUsage", StatusCategory.PERFORMANCE), + ("QueriesPeakMemoryUsage", StatusCategory.PERFORMANCE), + ("PSI_CPU_some", StatusCategory.PERFORMANCE), + ("PSI_IO_some", StatusCategory.PERFORMANCE), + ("PSI_MEM_some", StatusCategory.PERFORMANCE), + ("LongestRunningMerge", StatusCategory.PERFORMANCE), + ("NumberOfDatabases", StatusCategory.STORAGE), + ("NumberOfTables", StatusCategory.STORAGE), + ("TotalPartsOfMergeTreeTables", StatusCategory.STORAGE), + ("TotalRowsOfMergeTreeTables", StatusCategory.STORAGE), + ("TotalBytesOfMergeTreeTables", StatusCategory.STORAGE), + ("TotalUncompressedBytesOfMergeTreeTables", StatusCategory.STORAGE), + ("TotalPrimaryKeyBytesInMemory", StatusCategory.STORAGE), + ("TotalIndexGranularityBytesInMemory", StatusCategory.STORAGE), + ("FilesystemCacheBytes", StatusCategory.STORAGE), + ("FilesystemCacheCapacity", StatusCategory.STORAGE), + ("FilesystemCacheFiles", StatusCategory.STORAGE), + ("ReplicasMaxQueueSize", StatusCategory.REPLICATION), + ("ReplicasMaxAbsoluteDelay", StatusCategory.REPLICATION), + ("ReplicasMaxRelativeDelay", StatusCategory.REPLICATION), +) + +# ``system.settings.type`` names these; anything else is a value type rather +# than a unit, so a byte count stays unitless rather than being guessed at. +_TIME_UNITS = frozenset({"Seconds", "Milliseconds", "Microseconds", "Nanoseconds"}) + +# The gauges that count connections currently open on the server. ClickHouse has +# no single ``Threads_connected`` analogue — these are its per-listener counts, +# and ``get_connection_info`` sums exactly this set. +OPEN_CONNECTION_METRICS: Tuple[str, ...] = ( + "HTTPConnection", + "TCPConnection", + "InterserverConnection", + "MySQLConnection", + "PostgreSQLConnection", +) + +# ClickHouse has no ``SUPER`` privilege and ``system.users`` carries no such +# column. An account is reported as ``is_superuser`` when it holds the whole +# administrative set, unrestricted: server control, access viewing, role +# administration and user creation. Measured on 26.7.3.19, ``root`` holds all +# four (they are granted individually rather than as ``access_type = 'ALL'``). +_SUPERUSER_ACCESS_TYPES: Tuple[str, ...] = ( + "SYSTEM", + "SHOW ACCESS", + "ROLE ADMIN", + "CREATE USER", +) class ClickHouseStatusIntrospectorMixin: - """Mixin providing shared ClickHouse status introspection logic.""" + """Mixin providing shared ClickHouse status introspection logic. + + Only non-I/O work lives here: value parsing and the assembly of + :class:`StatusItem` lists from rows the sync/async classes read. The two + concrete classes keep identical method names so the sync/async parity + contract holds. + """ def _get_vendor_name(self) -> str: """Get ClickHouse vendor name.""" @@ -156,10 +414,17 @@ def _parse_variable_value(self, value: Any) -> Any: def _parse_version_string(self, version_str: str) -> tuple: """Parse ClickHouse version string to (major, minor, patch) tuple. - Examples: - '9.6.0' -> (9, 6, 0) - '8.0.36' -> (8, 0, 36) - '5.7.42-log' -> (5, 7, 42) + ClickHouse numbers releases ``YY.M.patch``, so the major component is a + two-digit year. Examples, using the shape of the string this server + returns (``SELECT version()`` is ``'26.7.3.19'``) plus the build-suffix + form: + + '26.7.3.19' -> (26, 7, 3) + '25.8.4.13' -> (25, 8, 4) + '26.3.1.1-alpine' -> (26, 3, 1) + + The extra fourth component this server reports is dropped; the tuple is + three wide, as the callers expect. """ if not version_str: return (0, 0, 0) @@ -196,6 +461,9 @@ def _create_status_item( description: Optional[str] = None, unit: Optional[str] = None, is_readonly: bool = False, + is_dynamic: bool = True, + default_value: Any = None, + extra: Optional[Dict[str, Any]] = None, ) -> StatusItem: """Create a StatusItem with parsed value.""" return StatusItem( @@ -205,8 +473,182 @@ def _create_status_item( description=description, unit=unit, is_readonly=is_readonly, + is_dynamic=is_dynamic, + default_value=default_value, + extra=dict(extra) if extra else {}, ) + # ------------------------------------------------------------------ # + # Inventory assembly (pure: rows in, StatusItems out) + # ------------------------------------------------------------------ # + + @staticmethod + def _setting_unit(declared_type: Any) -> Optional[str]: + """Unit for a ``system.settings`` value type, if the type names one. + + ``system.settings.type`` is ``Seconds`` / ``Milliseconds`` for the + timeout settings and ``UInt64`` / ``Bool`` / ``MaxThreads`` / ... for + everything else, so only the former yields a unit. A byte-valued + setting reports ``UInt64`` and is therefore left unitless rather than + being labelled "bytes" on this backend's own authority. + """ + return declared_type if declared_type in _TIME_UNITS else None + + def _build_setting_items( + self, + rows: Iterable[Dict[str, Any]], + inventory: Sequence[Tuple[str, StatusCategory]], + source: str, + read_only_from: str, + ) -> List[StatusItem]: + """Assemble StatusItems for one configuration table. + + ``read_only_from`` names the column that answers "may this change + without a restart": ``readonly`` on ``system.settings`` (which the + server documents as "the current user cannot change the setting") or + ``changeable_without_restart`` on ``system.server_settings``. + """ + by_name = {row.get("name"): row for row in rows if row.get("name")} + items: List[StatusItem] = [] + for name, category in inventory: + row = by_name.get(name) + if row is None: + # This version of ClickHouse does not offer the name. Reporting + # a placeholder would be a claim about a server that did not + # make it, so the name is simply not reported. + continue + is_readonly = self._read_only_flag(row, read_only_from) + items.append( + self._create_status_item( + name=name, + value=row.get("value"), + category=category, + description=row.get("description"), + unit=self._setting_unit(row.get("type")), + is_readonly=is_readonly, + is_dynamic=not is_readonly, + default_value=row.get("default"), + extra={ + "source": source, + "type": row.get("type"), + "changed": bool(row.get("changed")), + }, + ) + ) + return items + + @staticmethod + def _read_only_flag(row: Dict[str, Any], column: str) -> bool: + """Read the read-only flag from whichever column the table uses.""" + if column == "readonly": + return bool(row.get("readonly")) + # ``changeable_without_restart`` is an enum: 'No' / 'IncreaseOnly' / + # 'DecreaseOnly' / 'Yes'. Only 'No' means the value is fixed. + return str(row.get("changeable_without_restart") or "").strip().lower() == "no" + + def _build_gauge_items( + self, + rows: Iterable[Dict[str, Any]], + inventory: Sequence[Tuple[str, StatusCategory]], + source: str, + ) -> List[StatusItem]: + """Assemble StatusItems for a ``system.metrics`` gauge. + + A unit is only claimed for the gauges whose name the server itself ends + in ``Bytes``; a gauge named ``MemoryTracking`` is bytes too, but the + server does not say so, so it stays unitless. + + A name this version does not publish is not reported: ``QueryNonInternal`` + and ``MemoryReservationDemand`` are 26.3+ names, so on 25.8 they are + absent from ``system.metrics`` and absent from the result. + """ + by_name = {row.get("metric"): row for row in rows if row.get("metric")} + items: List[StatusItem] = [] + for name, category in inventory: + row = by_name.get(name) + if row is None: + continue + items.append( + self._create_status_item( + name=name, + value=row.get("value"), + category=category, + description=row.get("description"), + unit="bytes" if name.endswith("Bytes") else None, + is_readonly=True, + is_dynamic=False, + extra={"source": source}, + ) + ) + return items + + def _build_event_items( + self, + rows: Iterable[Dict[str, Any]], + inventory: Sequence[Tuple[str, StatusCategory]], + source: str, + ) -> List[StatusItem]: + """Assemble StatusItems for a ``system.events`` counter.""" + by_name = {row.get("event"): row for row in rows if row.get("event")} + items: List[StatusItem] = [] + for name, category in inventory: + row = by_name.get(name) + if row is None: + continue + items.append( + self._create_status_item( + name=name, + value=row.get("value"), + category=category, + description=row.get("description"), + unit="microseconds" if name.endswith("Microseconds") else None, + is_readonly=True, + is_dynamic=False, + extra={"source": source}, + ) + ) + return items + + def _build_async_metric_items( + self, + rows: Iterable[Dict[str, Any]], + inventory: Sequence[Tuple[str, StatusCategory]], + source: str, + ) -> List[StatusItem]: + """Assemble StatusItems for a ``system.asynchronous_metrics`` sample.""" + by_name = {row.get("metric"): row for row in rows if row.get("metric")} + items: List[StatusItem] = [] + for name, category in inventory: + row = by_name.get(name) + if row is None: + continue + items.append( + self._create_status_item( + name=name, + value=row.get("value"), + category=category, + description=row.get("description"), + is_readonly=True, + is_dynamic=False, + extra={"source": source}, + ) + ) + return items + + @staticmethod + def _filter_category(items: List[StatusItem], category: Optional[StatusCategory]) -> List[StatusItem]: + if category is None: + return items + return [item for item in items if item.category == category] + + @staticmethod + def _placeholders(count: int, backend: Any) -> str: + return ", ".join([backend.dialect.p()] * count) + + # ------------------------------------------------------------------ # + # Overview assembly + # ------------------------------------------------------------------ # + def _build_server_overview( self, configuration: List[StatusItem], @@ -217,15 +659,18 @@ def _build_server_overview( users: List[UserInfo], version: str, session: Optional[SessionInfo] = None, - innodb: Optional[InnoDBInfo] = None, - binary_log: Optional[BinaryLogInfo] = None, processes: Optional[List[ProcessInfo]] = None, - slow_query: Optional[SlowQueryInfo] = None, ) -> ServerOverview: """Build ServerOverview from collected data. - ClickHouse-specific replication details are exposed through - ``system.replicas`` in ``extra`` rather than a dedicated dataclass. + ``ServerOverview`` carries three MySQL-named slots that this backend + never fills: ``innodb`` / ``binary_log`` / ``slow_query``. They are core + fields, so they are left at their dataclass default rather than being + passed as ``None`` through a parameter that would name them here too. + ClickHouse's own storage-engine and replication detail is reported in + ``extra`` — a ``ReplacingMergeTree`` buffer pool, a binary log and a + slow-query log do not exist, and ``system.replicas`` / ``system.parts`` + are where the equivalents live. """ replication_summary = None try: @@ -247,10 +692,7 @@ def _build_server_overview( storage=storage, databases=databases, users=users, - innodb=innodb, - binary_log=binary_log, processes=processes or [], - slow_query=slow_query, extra={"replicas": replication_summary} if replication_summary else {}, ) @@ -258,7 +700,9 @@ def _build_server_overview( class SyncClickHouseStatusIntrospector(ClickHouseStatusIntrospectorMixin, SyncAbstractStatusIntrospector): """Synchronous ClickHouse status introspector. - Uses SHOW VARIABLES and SHOW STATUS to gather server information. + Reads configuration from ``system.settings`` / ``system.server_settings`` + and metrics from ``system.metrics``, ``system.events`` and + ``system.asynchronous_metrics``. Usage:: @@ -273,11 +717,7 @@ def __init__(self, backend: Any) -> None: self._show = backend.introspector.show def get_overview(self) -> ServerOverview: - """Get complete ClickHouse status overview. - - MySQL-specific sections (InnoDB, binary log, slow queries) are not - applicable to ClickHouse and degrade to ``None`` instead of failing. - """ + """Get complete ClickHouse status overview.""" configuration = self.list_configuration() performance = self.list_performance_metrics() connections = self.get_connection_info() @@ -298,122 +738,175 @@ def get_overview(self) -> ServerOverview: users=users, version=version, session=session, - innodb=None, - binary_log=None, processes=processes, - slow_query=None, ) def _get_version_string(self) -> str: - """Get ClickHouse version string.""" + """Get ClickHouse version string. + + Prefers ``SELECT version()``, which is the authoritative answer. The + fallback is the backend's adapted version when there is one, and the + empty string when there is not — never an invented release, so a caller + cannot mistake a guess for a server-reported version. + """ try: result = self._backend.execute("SELECT version()") if result.data and result.data[0]: return str(next(iter(result.data[0].values()))) except Exception: pass - version_tuple = getattr(self._backend, "_version", (8, 0, 0)) + version_tuple = getattr(self._backend, "_version", None) + if not version_tuple: + return "" return ".".join(str(v) for v in version_tuple) - def list_configuration(self, category: Optional[StatusCategory] = None) -> List[StatusItem]: - """List ClickHouse configuration parameters via ``system.settings``.""" - items = [] - - var_dict = {} + def _rows(self, sql: str, params: Optional[tuple] = None) -> List[Dict[str, Any]]: + """Run a read and return its rows, or ``[]`` if the server refuses.""" try: - result = self._backend.execute( - "SELECT name, value, changed FROM system.settings" - ) - for row in result.data or []: - var_dict[row.get("name")] = row.get("value") + result = self._backend.execute(sql, params if params is not None else ()) + return list(result.data or []) except Exception: - return items + return [] - # Build status items for known variables - for var_name, var_category, description, unit, is_readonly in CLICKHOUSE_CONFIG_VARIABLES: - if category and var_category != category: - continue + def list_configuration(self, category: Optional[StatusCategory] = None) -> List[StatusItem]: + """List ClickHouse configuration parameters. - if var_name in var_dict: - item = self._create_status_item( - name=var_name, - value=var_dict[var_name], - category=var_category, - description=description, - unit=unit, - is_readonly=is_readonly, - ) - items.append(item) + ClickHouse splits its configuration in two, so both halves are read: + ``system.settings`` for what a session or query can change, and + ``system.server_settings`` for the ``config.xml`` values that need a + restart. Each item's ``extra['source']`` says which it came from. - return items + The name set follows the server: ``system.server_settings`` on 25.8 has + no ``path``, ``tmp_path`` or ``listen_backlog`` row, so those are not + reported there. Each item's value, default, description, unit and + read-only flag are the row's own columns. + """ + items: List[StatusItem] = [] + + settings_rows = self._rows( + "SELECT name, value, description, default, readonly, changed, type " + "FROM system.settings WHERE name IN (" + + self._placeholders(len(CLICKHOUSE_CONFIG_SETTINGS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_CONFIG_SETTINGS), + ) + items.extend( + self._build_setting_items( + settings_rows, CLICKHOUSE_CONFIG_SETTINGS, "system.settings", "readonly" + ) + ) + + server_rows = self._rows( + "SELECT name, value, description, default, changed, type, changeable_without_restart " + "FROM system.server_settings WHERE name IN (" + + self._placeholders(len(CLICKHOUSE_SERVER_SETTINGS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_SERVER_SETTINGS), + ) + items.extend( + self._build_setting_items( + server_rows, + CLICKHOUSE_SERVER_SETTINGS, + "system.server_settings", + "changeable_without_restart", + ) + ) + + return self._filter_category(items, category) def list_performance_metrics(self, category: Optional[StatusCategory] = None) -> List[StatusItem]: - """List ClickHouse performance metrics via ``system.metrics``.""" - items = [] + """List ClickHouse performance metrics. + + Three tables, because ClickHouse reports three different things: + ``system.metrics`` holds instantaneous gauges, ``system.events`` holds + monotonic counters since server start, and ``system.asynchronous_metrics`` + holds periodically sampled OS / disk figures. + + What comes back is what *this* server has, out of the names this backend + asks about — never a name the server cannot produce. Two consequences a + caller should know: + + * the set is version-dependent, because the tables are; + * the ``system.events`` part is additionally **history**-dependent: a + counter only appears once that event has fired at least once, so a + freshly started server legitimately reports fewer events than one that + has been running merges, inserts and mutations. + """ + items: List[StatusItem] = [] - status_dict = {} - try: - result = self._backend.execute( - "SELECT metric, value FROM system.metrics" - ) - for row in result.data or []: - status_dict[row.get("metric")] = row.get("value") - except Exception: - return items + gauge_rows = self._rows( + "SELECT metric, value, description FROM system.metrics WHERE metric IN (" + + self._placeholders(len(CLICKHOUSE_GAUGE_METRICS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_GAUGE_METRICS), + ) + items.extend(self._build_gauge_items(gauge_rows, CLICKHOUSE_GAUGE_METRICS, "system.metrics")) - # Build status items for known status variables - for var_name, var_category, description, unit in CLICKHOUSE_STATUS_VARIABLES: - if category and var_category != category: - continue + event_rows = self._rows( + "SELECT event, value, description FROM system.events WHERE event IN (" + + self._placeholders(len(CLICKHOUSE_COUNTER_EVENTS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_COUNTER_EVENTS), + ) + items.extend(self._build_event_items(event_rows, CLICKHOUSE_COUNTER_EVENTS, "system.events")) - if var_name in status_dict: - item = self._create_status_item( - name=var_name, - value=status_dict[var_name], - category=var_category, - description=description, - unit=unit, - ) - items.append(item) + async_rows = self._rows( + "SELECT metric, value, description FROM system.asynchronous_metrics WHERE metric IN (" + + self._placeholders(len(CLICKHOUSE_ASYNC_METRICS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_ASYNC_METRICS), + ) + items.extend( + self._build_async_metric_items(async_rows, CLICKHOUSE_ASYNC_METRICS, "system.asynchronous_metrics") + ) - return items + return self._filter_category(items, category) def get_connection_info(self) -> ConnectionInfo: - """Get connection information from ``system.metrics`` / ``system.settings``.""" - status_dict = {} - try: - result = self._backend.execute( - "SELECT metric, value FROM system.metrics " - "WHERE metric IN ('Connection', 'HTTPConnection', 'TCPConnection', 'InterserverConnection')" - ) - for row in result.data or []: - status_dict[row.get("metric")] = row.get("value") - except Exception: - pass + """Get connection information from ``system.metrics`` / ``system.server_settings``. - var_dict = {} - try: - result = self._backend.execute( - "SELECT value FROM system.settings WHERE name = 'max_connections'" - ) - if result.data: - var_dict["max_connections"] = next(iter(result.data[0].values())) - except Exception: - pass + ``max_connections`` is a ``config.xml`` value, so it is read from + ``system.server_settings``; a ``system.settings`` lookup for that name + returns no row at all. ``active_count`` sums the five per-listener + gauges in :data:`OPEN_CONNECTION_METRICS`, because ClickHouse has no + single ``Threads_connected`` to report. + """ + gauge_rows = self._rows( + "SELECT metric, value FROM system.metrics WHERE metric IN (" + + self._placeholders(len(OPEN_CONNECTION_METRICS), self._backend) + + ")", + OPEN_CONNECTION_METRICS, + ) + gauges = {row.get("metric"): row.get("value") for row in gauge_rows} - active = sum( - self._parse_variable_value(v) or 0 - for k, v in status_dict.items() - if "connection" in k.lower() + server_rows = self._rows( + "SELECT name, value FROM system.server_settings " + f"WHERE name = {self._backend.dialect.p()}", + ("max_connections",), ) + max_connections = server_rows[0].get("value") if server_rows else None + + def _num(name: str) -> Optional[int]: + value = self._parse_variable_value(gauges.get(name)) + return value if isinstance(value, int) else None + + active = [v for v in (_num(name) for name in OPEN_CONNECTION_METRICS) if v is not None] return ConnectionInfo( - active_count=active or None, - max_connections=self._parse_variable_value(var_dict.get("max_connections")), + active_count=sum(active) if active else None, + max_connections=self._parse_variable_value(max_connections), idle_count=None, extra={ - "http_connections": self._parse_variable_value(status_dict.get("HTTPConnection")), - "tcp_connections": self._parse_variable_value(status_dict.get("TCPConnection")), + "http_connections": _num("HTTPConnection"), + "tcp_connections": _num("TCPConnection"), + "interserver_connections": _num("InterserverConnection"), + "mysql_connections": _num("MySQLConnection"), + "postgresql_connections": _num("PostgreSQLConnection"), + "note": ( + "ClickHouse reports one gauge per listener and no combined " + "'Threads_connected'; active_count is the sum of " + + ", ".join(OPEN_CONNECTION_METRICS) + ), }, ) @@ -431,40 +924,45 @@ def get_storage_info(self) -> StorageInfo: except Exception: pass - datadir = None - buffer_size = None - try: - result = self._backend.execute( - "SELECT path, free_space FROM system.disks WHERE name = 'default'" - ) - if result.data: - datadir = result.data[0].get("path") - buffer_size = result.data[0].get("free_space") - except Exception: - pass + disks = self._rows( + "SELECT name, path, type, total_space, free_space, unreserved_space, keep_free_space, " + "is_encrypted, is_read_only, is_remote, is_broken FROM system.disks" + ) + default_disk = next((d for d in disks if d.get("name") == "default"), None) return StorageInfo( total_size_bytes=self._parse_variable_value(total_size), + free_space_bytes=self._parse_variable_value(default_disk.get("free_space")) if default_disk else None, extra={ - "datadir": datadir, - "free_space": self._parse_variable_value(buffer_size), + "database": self._backend.config.database, + "data_path": default_disk.get("path") if default_disk else None, + "disks": disks, }, ) def list_databases(self) -> List[DatabaseBriefInfo]: - """List databases with table/view counts.""" + """List databases with table/view counts. + + The counts are whatever ``information_schema.TABLES`` says, so a + database nobody has created a table in reports ``0`` — a fresh + ClickHouse container's ``default`` database is empty, and says so + truthfully. A count is never padded with a guess to look populated. + """ databases = [] db_results = self._show.databases() db_names = [db.name for db in db_results] - # Get table and view counts for all databases from information_schema + # Get table and view counts for all databases from information_schema. + # ClickHouse exposes both spellings of every information_schema column + # (measured on 26.7.3.19: ``table_schema`` and ``TABLE_SCHEMA`` are both + # returned), and its ``table_type`` uses the same ``BASE TABLE`` / + # ``VIEW`` values MySQL does. table_counts: Dict[str, int] = {} view_counts: Dict[str, int] = {} try: - p = self._backend.dialect.p() - placeholders = ", ".join([p] * len(db_names)) + placeholders = self._placeholders(len(db_names), self._backend) result = self._backend.execute( "SELECT table_schema, table_type, COUNT(*) as count " "FROM information_schema.TABLES " @@ -474,9 +972,8 @@ def list_databases(self) -> List[DatabaseBriefInfo]: ) if result and result.data: for row in result.data: - # ClickHouse returns column names in uppercase - schema = row.get("TABLE_SCHEMA") or row.get("table_schema") - table_type = row.get("TABLE_TYPE") or row.get("table_type") + schema = row.get("table_schema") or row.get("TABLE_SCHEMA") + table_type = row.get("table_type") or row.get("TABLE_TYPE") count = row.get("count", 0) or row.get("COUNT", 0) if table_type == "BASE TABLE": table_counts[schema] = count @@ -496,22 +993,62 @@ def list_databases(self) -> List[DatabaseBriefInfo]: return databases def list_users(self) -> List[UserInfo]: - """List users from clickhouse.user table.""" + """List users from ``system.users`` with their grants and roles. + + There is no ``clickhouse.user`` database on a stock server (measured on + 26.7.3.19: ``UNKNOWN_DATABASE``), so ``system.users`` is the source. + It carries no superuser column, so ``is_superuser`` is derived from the + account's grants against ``system.grants`` and the full set lands in + ``extra['access_types']``. + """ users = [] - try: - result = self._backend.execute("SELECT User, Host, Super_priv FROM clickhouse.user", ()) - if result and result.data: - for row in result.data: - user = UserInfo( - name=row.get("User"), - host=row.get("Host"), - is_superuser=row.get("Super_priv") == "Y", - ) - users.append(user) - except Exception: - # clickhouse.user may not be accessible - pass + user_rows = self._rows( + "SELECT name, id, auth_type, default_database, host_ip, host_names FROM system.users" + ) + if not user_rows: + return users + + grant_rows = self._rows( + "SELECT user_name, access_type, database, table, grant_option FROM system.grants " + "WHERE database IS NULL AND table IS NULL" + ) + grants_by_user: Dict[str, List[str]] = {} + for row in grant_rows: + name = row.get("user_name") + access_type = row.get("access_type") + if name and access_type: + grants_by_user.setdefault(name, []).append(str(access_type)) + + role_rows = self._rows("SELECT user_name, role_name, granted_role_is_default FROM system.role_grants") + roles_by_user: Dict[str, List[str]] = {} + for row in role_rows: + name = row.get("user_name") + role_name = row.get("role_name") + if name and role_name: + roles_by_user.setdefault(name, []).append(str(role_name)) + + for row in user_rows: + name = row.get("name") + if not name: + continue + access_types = sorted(set(grants_by_user.get(name, []))) + users.append( + UserInfo( + name=name, + roles=sorted(set(roles_by_user.get(name, []))), + is_superuser=all(t in access_types for t in _SUPERUSER_ACCESS_TYPES), + extra={ + "id": str(row.get("id")), + "auth_type": list(row.get("auth_type") or []), + "default_database": row.get("default_database") or None, + "host_ip": list(row.get("host_ip") or []), + "host_names": list(row.get("host_names") or []), + "access_types": access_types, + "superuser_test": list(_SUPERUSER_ACCESS_TYPES), + }, + ) + ) return users @@ -540,41 +1077,38 @@ def get_session_info(self) -> SessionInfo: return session - - def list_processes(self) -> List[ProcessInfo]: """List current running processes/queries via ``system.processes``.""" processes = [] - try: - result = self._backend.execute( - "SELECT query_id, user, address, currentDatabase, elapsed, query " - "FROM system.processes", - (), + rows = self._rows( + "SELECT query_id, user, address, current_database, elapsed, query " + "FROM system.processes" + ) + for row in rows: + processes.append( + ProcessInfo( + id=row.get("query_id"), + user=row.get("user"), + host=str(row.get("address") or "") or None, + # ClickHouse spells this ``current_database``; MySQL's + # ``currentDatabase`` is not a column here. + database=row.get("current_database"), + command="QUERY", + time=row.get("elapsed"), + state=None, + info=row.get("query"), + ) ) - if result and result.data: - for row in result.data: - proc = ProcessInfo( - id=row.get("query_id"), - user=row.get("user"), - host=str(row.get("address") or "") or None, - database=row.get("currentDatabase"), - command="QUERY", - time=row.get("elapsed"), - state=None, - info=row.get("query"), - ) - processes.append(proc) - except Exception: - pass return processes - - class AsyncClickHouseStatusIntrospector(ClickHouseStatusIntrospectorMixin, AsyncAbstractStatusIntrospector): """Asynchronous ClickHouse status introspector. - Uses SHOW VARIABLES and SHOW STATUS to gather server information. + Reads configuration from ``system.settings`` / ``system.server_settings`` + and metrics from ``system.metrics``, ``system.events`` and + ``system.asynchronous_metrics``. Same inventories and same method names as + the synchronous class; only the awaits differ. Usage:: @@ -589,11 +1123,7 @@ def __init__(self, backend: Any) -> None: self._show = backend.introspector.show async def get_overview(self) -> ServerOverview: - """Get complete ClickHouse status overview. - - MySQL-specific sections (InnoDB, binary log, slow queries) are not - applicable to ClickHouse and degrade to ``None`` instead of failing. - """ + """Get complete ClickHouse status overview.""" configuration = await self.list_configuration() performance = await self.list_performance_metrics() connections = await self.get_connection_info() @@ -614,120 +1144,159 @@ async def get_overview(self) -> ServerOverview: users=users, version=version, session=session, - innodb=None, - binary_log=None, processes=processes, - slow_query=None, ) async def _get_version_string(self) -> str: - """Get ClickHouse version string.""" + """Get ClickHouse version string — see the sync twin for the fallback rule.""" try: result = await self._backend.execute("SELECT version()") if result.data and result.data[0]: return str(next(iter(result.data[0].values()))) except Exception: pass - version_tuple = getattr(self._backend, "_version", (8, 0, 0)) + version_tuple = getattr(self._backend, "_version", None) + if not version_tuple: + return "" return ".".join(str(v) for v in version_tuple) - async def list_configuration(self, category: Optional[StatusCategory] = None) -> List[StatusItem]: - """List ClickHouse configuration parameters via ``system.settings``.""" - items = [] - - var_dict = {} + async def _rows(self, sql: str, params: Optional[tuple] = None) -> List[Dict[str, Any]]: + """Run a read and return its rows, or ``[]`` if the server refuses.""" try: - result = await self._backend.execute( - "SELECT name, value, changed FROM system.settings" - ) - for row in result.data or []: - var_dict[row.get("name")] = row.get("value") + result = await self._backend.execute(sql, params if params is not None else ()) + return list(result.data or []) except Exception: - return items + return [] - for var_name, var_category, description, unit, is_readonly in CLICKHOUSE_CONFIG_VARIABLES: - if category and var_category != category: - continue + async def list_configuration(self, category: Optional[StatusCategory] = None) -> List[StatusItem]: + """List ClickHouse configuration parameters. - if var_name in var_dict: - item = self._create_status_item( - name=var_name, - value=var_dict[var_name], - category=var_category, - description=description, - unit=unit, - is_readonly=is_readonly, - ) - items.append(item) + ClickHouse splits its configuration in two, so both halves are read: + ``system.settings`` for what a session or query can change, and + ``system.server_settings`` for the ``config.xml`` values that need a + restart. Each item's ``extra['source']`` says which it came from. - return items + See the synchronous twin for why the name set follows the server. + """ + items: List[StatusItem] = [] + + settings_rows = await self._rows( + "SELECT name, value, description, default, readonly, changed, type " + "FROM system.settings WHERE name IN (" + + self._placeholders(len(CLICKHOUSE_CONFIG_SETTINGS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_CONFIG_SETTINGS), + ) + items.extend( + self._build_setting_items( + settings_rows, CLICKHOUSE_CONFIG_SETTINGS, "system.settings", "readonly" + ) + ) + + server_rows = await self._rows( + "SELECT name, value, description, default, changed, type, changeable_without_restart " + "FROM system.server_settings WHERE name IN (" + + self._placeholders(len(CLICKHOUSE_SERVER_SETTINGS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_SERVER_SETTINGS), + ) + items.extend( + self._build_setting_items( + server_rows, + CLICKHOUSE_SERVER_SETTINGS, + "system.server_settings", + "changeable_without_restart", + ) + ) + + return self._filter_category(items, category) async def list_performance_metrics(self, category: Optional[StatusCategory] = None) -> List[StatusItem]: - """List ClickHouse performance metrics via ``system.metrics``.""" - items = [] + """List ClickHouse performance metrics. - status_dict = {} - try: - result = await self._backend.execute( - "SELECT metric, value FROM system.metrics" - ) - for row in result.data or []: - status_dict[row.get("metric")] = row.get("value") - except Exception: - return items + Three tables, because ClickHouse reports three different things: + ``system.metrics`` holds instantaneous gauges, ``system.events`` holds + monotonic counters since server start, and ``system.asynchronous_metrics`` + holds periodically sampled OS / disk figures. - for var_name, var_category, description, unit in CLICKHOUSE_STATUS_VARIABLES: - if category and var_category != category: - continue + See the synchronous twin for why the set is version- and + history-dependent. + """ + items: List[StatusItem] = [] - if var_name in status_dict: - item = self._create_status_item( - name=var_name, - value=status_dict[var_name], - category=var_category, - description=description, - unit=unit, - ) - items.append(item) + gauge_rows = await self._rows( + "SELECT metric, value, description FROM system.metrics WHERE metric IN (" + + self._placeholders(len(CLICKHOUSE_GAUGE_METRICS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_GAUGE_METRICS), + ) + items.extend(self._build_gauge_items(gauge_rows, CLICKHOUSE_GAUGE_METRICS, "system.metrics")) - return items + event_rows = await self._rows( + "SELECT event, value, description FROM system.events WHERE event IN (" + + self._placeholders(len(CLICKHOUSE_COUNTER_EVENTS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_COUNTER_EVENTS), + ) + items.extend(self._build_event_items(event_rows, CLICKHOUSE_COUNTER_EVENTS, "system.events")) + + async_rows = await self._rows( + "SELECT metric, value, description FROM system.asynchronous_metrics WHERE metric IN (" + + self._placeholders(len(CLICKHOUSE_ASYNC_METRICS), self._backend) + + ")", + tuple(name for name, _ in CLICKHOUSE_ASYNC_METRICS), + ) + items.extend( + self._build_async_metric_items(async_rows, CLICKHOUSE_ASYNC_METRICS, "system.asynchronous_metrics") + ) + + return self._filter_category(items, category) async def get_connection_info(self) -> ConnectionInfo: - """Get connection information from ``system.metrics`` / ``system.settings``.""" - status_dict = {} - try: - result = await self._backend.execute( - "SELECT metric, value FROM system.metrics " - "WHERE metric IN ('Connection', 'HTTPConnection', 'TCPConnection', 'InterserverConnection')" - ) - for row in result.data or []: - status_dict[row.get("metric")] = row.get("value") - except Exception: - pass + """Get connection information from ``system.metrics`` / ``system.server_settings``. - var_dict = {} - try: - result = await self._backend.execute( - "SELECT value FROM system.settings WHERE name = 'max_connections'" - ) - if result.data: - var_dict["max_connections"] = next(iter(result.data[0].values())) - except Exception: - pass + ``max_connections`` is a ``config.xml`` value, so it is read from + ``system.server_settings``; a ``system.settings`` lookup for that name + returns no row at all. ``active_count`` sums the five per-listener + gauges in :data:`OPEN_CONNECTION_METRICS`, because ClickHouse has no + single ``Threads_connected`` to report. + """ + gauge_rows = await self._rows( + "SELECT metric, value FROM system.metrics WHERE metric IN (" + + self._placeholders(len(OPEN_CONNECTION_METRICS), self._backend) + + ")", + OPEN_CONNECTION_METRICS, + ) + gauges = {row.get("metric"): row.get("value") for row in gauge_rows} - active = sum( - self._parse_variable_value(v) or 0 - for k, v in status_dict.items() - if "connection" in k.lower() + server_rows = await self._rows( + "SELECT name, value FROM system.server_settings " + f"WHERE name = {self._backend.dialect.p()}", + ("max_connections",), ) + max_connections = server_rows[0].get("value") if server_rows else None + + def _num(name: str) -> Optional[int]: + value = self._parse_variable_value(gauges.get(name)) + return value if isinstance(value, int) else None + + active = [v for v in (_num(name) for name in OPEN_CONNECTION_METRICS) if v is not None] return ConnectionInfo( - active_count=active or None, - max_connections=self._parse_variable_value(var_dict.get("max_connections")), + active_count=sum(active) if active else None, + max_connections=self._parse_variable_value(max_connections), idle_count=None, extra={ - "http_connections": self._parse_variable_value(status_dict.get("HTTPConnection")), - "tcp_connections": self._parse_variable_value(status_dict.get("TCPConnection")), + "http_connections": _num("HTTPConnection"), + "tcp_connections": _num("TCPConnection"), + "interserver_connections": _num("InterserverConnection"), + "mysql_connections": _num("MySQLConnection"), + "postgresql_connections": _num("PostgreSQLConnection"), + "note": ( + "ClickHouse reports one gauge per listener and no combined " + "'Threads_connected'; active_count is the sum of " + + ", ".join(OPEN_CONNECTION_METRICS) + ), }, ) @@ -745,23 +1314,19 @@ async def get_storage_info(self) -> StorageInfo: except Exception: pass - datadir = None - free_space = None - try: - result = await self._backend.execute( - "SELECT path, free_space FROM system.disks WHERE name = 'default'" - ) - if result.data: - datadir = result.data[0].get("path") - free_space = result.data[0].get("free_space") - except Exception: - pass + disks = await self._rows( + "SELECT name, path, type, total_space, free_space, unreserved_space, keep_free_space, " + "is_encrypted, is_read_only, is_remote, is_broken FROM system.disks" + ) + default_disk = next((d for d in disks if d.get("name") == "default"), None) return StorageInfo( total_size_bytes=self._parse_variable_value(total_size), + free_space_bytes=self._parse_variable_value(default_disk.get("free_space")) if default_disk else None, extra={ - "datadir": datadir, - "free_space": self._parse_variable_value(free_space), + "database": self._backend.config.database, + "data_path": default_disk.get("path") if default_disk else None, + "disks": disks, }, ) @@ -772,13 +1337,16 @@ async def list_databases(self) -> List[DatabaseBriefInfo]: db_results = await self._show.databases() db_names = [db.name for db in db_results] - # Get table and view counts for all databases from information_schema + # Get table and view counts for all databases from information_schema. + # ClickHouse exposes both spellings of every information_schema column + # (measured on 26.7.3.19: ``table_schema`` and ``TABLE_SCHEMA`` are both + # returned), and its ``table_type`` uses the same ``BASE TABLE`` / + # ``VIEW`` values MySQL does. table_counts: Dict[str, int] = {} view_counts: Dict[str, int] = {} try: - p = self._backend.dialect.p() - placeholders = ", ".join([p] * len(db_names)) + placeholders = self._placeholders(len(db_names), self._backend) result = await self._backend.execute( "SELECT table_schema, table_type, COUNT(*) as count " "FROM information_schema.TABLES " @@ -788,9 +1356,8 @@ async def list_databases(self) -> List[DatabaseBriefInfo]: ) if result and result.data: for row in result.data: - # ClickHouse returns column names in uppercase - schema = row.get("TABLE_SCHEMA") or row.get("table_schema") - table_type = row.get("TABLE_TYPE") or row.get("table_type") + schema = row.get("table_schema") or row.get("TABLE_SCHEMA") + table_type = row.get("table_type") or row.get("TABLE_TYPE") count = row.get("count", 0) or row.get("COUNT", 0) if table_type == "BASE TABLE": table_counts[schema] = count @@ -810,21 +1377,62 @@ async def list_databases(self) -> List[DatabaseBriefInfo]: return databases async def list_users(self) -> List[UserInfo]: - """List users from clickhouse.user table.""" + """List users from ``system.users`` with their grants and roles. + + There is no ``clickhouse.user`` database on a stock server (measured on + 26.7.3.19: ``UNKNOWN_DATABASE``), so ``system.users`` is the source. + It carries no superuser column, so ``is_superuser`` is derived from the + account's grants against ``system.grants`` and the full set lands in + ``extra['access_types']``. + """ users = [] - try: - result = await self._backend.execute("SELECT User, Host, Super_priv FROM clickhouse.user", ()) - if result and result.data: - for row in result.data: - user = UserInfo( - name=row.get("User"), - host=row.get("Host"), - is_superuser=row.get("Super_priv") == "Y", - ) - users.append(user) - except Exception: - pass + user_rows = await self._rows( + "SELECT name, id, auth_type, default_database, host_ip, host_names FROM system.users" + ) + if not user_rows: + return users + + grant_rows = await self._rows( + "SELECT user_name, access_type, database, table, grant_option FROM system.grants " + "WHERE database IS NULL AND table IS NULL" + ) + grants_by_user: Dict[str, List[str]] = {} + for row in grant_rows: + name = row.get("user_name") + access_type = row.get("access_type") + if name and access_type: + grants_by_user.setdefault(name, []).append(str(access_type)) + + role_rows = await self._rows("SELECT user_name, role_name, granted_role_is_default FROM system.role_grants") + roles_by_user: Dict[str, List[str]] = {} + for row in role_rows: + name = row.get("user_name") + role_name = row.get("role_name") + if name and role_name: + roles_by_user.setdefault(name, []).append(str(role_name)) + + for row in user_rows: + name = row.get("name") + if not name: + continue + access_types = sorted(set(grants_by_user.get(name, []))) + users.append( + UserInfo( + name=name, + roles=sorted(set(roles_by_user.get(name, []))), + is_superuser=all(t in access_types for t in _SUPERUSER_ACCESS_TYPES), + extra={ + "id": str(row.get("id")), + "auth_type": list(row.get("auth_type") or []), + "default_database": row.get("default_database") or None, + "host_ip": list(row.get("host_ip") or []), + "host_names": list(row.get("host_names") or []), + "access_types": access_types, + "superuser_test": list(_SUPERUSER_ACCESS_TYPES), + }, + ) + ) return users @@ -848,37 +1456,31 @@ async def get_session_info(self) -> SessionInfo: # The HTTP interface reports no TLS session details to the client; # ssl_enabled stays at its default (None). - # Check if password was used + # Check if password was used (connection was made with password) session.password_used = bool(self._backend.config.password) return session - - async def list_processes(self) -> List[ProcessInfo]: """List current running processes/queries via ``system.processes``.""" processes = [] - try: - result = await self._backend.execute( - "SELECT query_id, user, address, currentDatabase, elapsed, query " - "FROM system.processes", - (), + rows = await self._rows( + "SELECT query_id, user, address, current_database, elapsed, query " + "FROM system.processes" + ) + for row in rows: + processes.append( + ProcessInfo( + id=row.get("query_id"), + user=row.get("user"), + host=str(row.get("address") or "") or None, + # ClickHouse spells this ``current_database``; MySQL's + # ``currentDatabase`` is not a column here. + database=row.get("current_database"), + command="QUERY", + time=row.get("elapsed"), + state=None, + info=row.get("query"), + ) ) - if result and result.data: - for row in result.data: - proc = ProcessInfo( - id=row.get("query_id"), - user=row.get("user"), - host=str(row.get("address") or "") or None, - database=row.get("currentDatabase"), - command="QUERY", - time=row.get("elapsed"), - state=None, - info=row.get("query"), - ) - processes.append(proc) - except Exception: - pass - return processes - - + return processes \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/__init__.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/__init__.py index 0da1929..02f45bd 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/__init__.py @@ -8,7 +8,6 @@ from .set_type import ClickHouseSetTypeMixin from .json import ClickHouseJSONFunctionMixin from .spatial import ClickHouseSpatialMixin -from .vector import ClickHouseVectorMixin from .dml import ClickHouseDMLOperationMixin from .fulltext import ClickHouseFullTextSearchMixin from .locking import ClickHouseLockingMixin @@ -52,6 +51,8 @@ from .constraint import ClickHouseConstraintMixin from .ddl_column import ClickHouseDDLColumnMixin from .function import ClickHouseFunctionMixin +from .column_type import ClickHouseColumnTypeMixin +from .trim import ClickHouseTrimMixin __all__ = [ "ClickHouseIntrospectionMixin", @@ -63,7 +64,6 @@ "ClickHouseSetTypeMixin", "ClickHouseJSONFunctionMixin", "ClickHouseSpatialMixin", - "ClickHouseVectorMixin", "ClickHouseDMLOperationMixin", "ClickHouseFullTextSearchMixin", "ClickHouseLockingMixin", @@ -106,4 +106,6 @@ "ClickHouseConstraintMixin", "ClickHouseDDLColumnMixin", "ClickHouseFunctionMixin", + "ClickHouseColumnTypeMixin", + "ClickHouseTrimMixin", ] diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/column_type.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/column_type.py new file mode 100644 index 0000000..0699751 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/column_type.py @@ -0,0 +1,229 @@ +# src/rhosocial/activerecord/backend/impl/clickhouse/mixins/column_type.py +"""Which column class ClickHouse suggests for each common Python type. + +This is the ClickHouse half of the column-type protocol: an answer for every +one of the eighteen common Python types the framework defines, and nothing +else. Everything here is a **column** decision -- what operations a value +carries -- and none of it is a storage decision: whether ``Array(T)`` or +``JSON`` spells the column is the DDL layer's separate answer (see +:class:`ClickHouseTypeSupportMixin`), and this table never reads it. + +Evidence +-------- +Every cell below is backed by a probe against a live **ClickHouse 26.7.3.19** +server (HTTP ``127.0.0.1:18682``, ``clickhouse-connect`` 1.9.0), recorded in the +core plan directory and, for the measured matrix, in this repository's +``.claude/plan/2026-10-08/secondary-gaps-investigation.md`` appendix C: + +* ``suggested-pairing-array.md`` -- the native-array column of the matrix; +* ``suggested-pairing-json.md`` -- the ``dict`` battery (J00-J15); +* ``suggested-pairing-string-enum.md`` -- ``str`` / ``bytes`` / ``enum``; +* ``suggested-pairing-numeric.md`` -- the numeric families. + +What ClickHouse is *not* given here +----------------------------------- +Two questions are deliberately left open rather than answered, because answering +them would be a ruling nobody has made: + +* **The ``UInt`` question (议题 C).** ClickHouse has a full unsigned family + (``UInt8``..``UInt256``) and "a column that can only hold a non-negative + number" is a real semantic difference. Whether that deserves a column class of + its own is undecided, so ``int`` answers :class:`IntegerColumn` like every + other backend. This table is not a ruling on 议题 C and must not be read as + one; it only says the question is still open. +* **``Tuple`` / ``Map`` column classes (议题 A).** ClickHouse's ``Tuple`` is a + heterogeneous positional group and ``Map`` is native, and core has no column + class for either. ``tuple`` is answered :class:`ArrayColumn` because that is + what the protocol's agreed ClickHouse row says and because the permissive + fallback this entry used to reach no longer exists; if 议题 A decides + that ``Tuple`` gets a class, this entry moves with it. + +Two rendering facts recorded here and **not** acted on, because rendering +belongs to Phase 2b: + +* ``LENGTH`` on a ClickHouse ``String`` returns **bytes** (``length(s) = 9`` for + five characters / nine UTF-8 bytes; ``lengthUTF8`` is the character count) -- + one of only two backends where that is true. A length operation must not + assume a character count. +* Equality on a ``JSON`` column has to render as ``toString(doc) = ?``; the + comparison exists and is not narrowed, but the bare ``=`` is not what the + server accepts. + +The JSON version gate +--------------------- +The native ``JSON`` data type first appears on ClickHouse 26.0 -- the version +this backend already declares for JSON elsewhere +(:meth:`ClickHouseJSONFunctionMixin.supports_json_type` and +``_JSON_FUNCTION_VERSIONS`` both answer ``26.0``). Below it the JSON surface is +carried by the ``JSONExtract*`` / ``JSONHas`` / ``isValidJSON`` function family +over ``String`` storage, which is why the ``dict`` entry is answerable on every +version the backend supports and never needs the protocol's ``None``. This is +**not** a measured boundary: the probe battery ran on 26.7.3.19 only, and no +older server was available to find the exact first version. What is measured is +that on 26.7 the native type needs no experimental setting and plain DDL +declares it. + +The capability mechanism this used to be routed through was removed from core: +nothing in production called it, and it had drifted from real behaviour. The +fact it carried survives here as a note, because a future phase that offers +``json_value`` has to know that the chainable document accessor needs the +native type. +""" + +import datetime +import decimal +import enum +import uuid +from typing import Any, Dict, Optional, Type + +from rhosocial.activerecord.backend.dialect.mixins import ColumnTypeMixin +from rhosocial.activerecord.backend.expression.column_types import ( + ArrayColumn, + BinaryColumn, + BooleanColumn, + ColumnBase, + IntegerColumn, + JSONColumn, + NumericColumn, + StringColumn, + TimestampColumn, + UUIDColumn, +) + +#: ClickHouse's full table: ``{common Python type: ColumnBase subclass}``. +#: +#: Every entry of the protocol's common-type list is answered, and every answer +#: is a column class: no entry is ``None``, and that is a finding rather than an +#: omission. ClickHouse has every capability the protocol's other backends lack +#: a default for: a native array, a native JSON type, and the ``ILIKE`` operator +#: that only PostgreSQL shares. The two ends that *do* answer ``None`` +#: elsewhere -- Firebird's ``dict`` and ``list`` (no JSON functions, no arrays at +#: all) and MySQL 5.6's (no JSON functions) -- have nothing here to refuse. +#: ``dict`` stays answerable below the JSON type's first version because the +#: ``JSONExtract`` family, not the type, is what carries it. +#: +#: There is no core table to derive from any more: the neutral common-type table +#: was removed with the rebuild, so these are this backend's own answers, stated +#: in full. The numeric family is answered with the one class core now carries +#: -- see the comment on the ``float`` entry. +CLICKHOUSE_COLUMN_TYPES: Dict[Any, Optional[Type[ColumnBase]]] = { + # --- numbers --------------------------------------------------------- + # The measured facts behind this cell are that §1 calls `float` the whole + # Float family here (Float32 / Float64 / BFloat16) and `Decimal` the + # Decimal32/64/128/256 family, that Float64 yields inf on overflow and + # carries nan natively, and that `AVG` of a Decimal answers Float64 rather + # than a Decimal (suggested-pairing-numeric.md). Those are real differences + # in what the server does, but core now carries one class for every numeric + # width and precision -- the operations are the same whatever the value was + # declared as, and the difference between a `Decimal(18, 4)` and a + # `Float64` belongs to the DDL layer's DataType. So both entries answer + # NumericColumn, and the width stays a rendering concern. + float: NumericColumn, + decimal.Decimal: NumericColumn, + # --- booleans ------------------------------------------------------ + # `Bool` storage, and ClickHouse's Bool is UInt8 underneath + # (suggested-mappings.md §1). The column class is unaffected: the + # boolean-domain surface (`is_true` / `is_false` / `&` / `|` / `~`) is + # BooleanColumn's, and core renders the literals as TRUE / FALSE. + bool: BooleanColumn, + # --- integers ------------------------------------------------------ + # The UInt question (议题 C) is open and this cell does not decide it: + # see the module docstring. `int` is a whole number here as anywhere. + int: IntegerColumn, + # --- text and bytes ------------------------------------------------ + # String, with no server-side collation concept (§5 of the string + # probe: ClickHouse compares bytes / code points, `Zebra` = `zebra` is + # false). `ilike` is **not** narrowed -- ClickHouse is one of only two + # backends that has it, and it was measured hitting both 'aXc' and + # 'ABC'. LENGTH returning bytes is a Phase 2b rendering fact, noted in + # the module docstring. + str: StringColumn, + # Bytes live in `String`, which is already byte-oriented; the driver + # hands them back as a hex string for non-UTF-8 payloads, a value-layer + # fact rather than a column-class one. Comparison, byte length, + # substring, concat and hex were all measured present. + bytes: BinaryColumn, + bytearray: BinaryColumn, + # --- date / time --------------------------------------------------- + # TimestampColumn for all three temporal entries, which is the shared + # baseline. Two are provisional and say so: core has no DateColumn yet + # (appendix C, `date`), and this backend's own TimeType renders DateTime + # even though ClickHouse has had Time/Time64 for a while (appendix D-①, + # 待办 #7). Neither gap is a reason to refuse an entry; both are reasons + # the answer may move. + datetime.date: TimestampColumn, + datetime.time: TimestampColumn, + datetime.datetime: TimestampColumn, + # A timedelta is a number of seconds here, answered by the numeric + # surface. ClickHouse does have a native `Interval` family and the + # driver binds a timedelta natively, but core has no IntervalColumn, and + # the native type's arithmetic surface is **not** measured: the + # aggregate fact table found `sum` / `avg` over `Interval*` rejected with + # ILLEGAL_TYPE_OF_ARGUMENT on 26.7.3.19 even though the docs promise a + # same-type result. So the honest answer is the numeric one, and the + # Interval question rides with core's missing class. + datetime.timedelta: NumericColumn, + # Native `UUID` type, compared as a value like any other: portable SQL + # has no UUID operator for a UUID column to be missing. + uuid.UUID: UUIDColumn, + # --- documents ------------------------------------------------------ + # Native `JSON` on 26.0+, and the `JSONExtract*` family over String + # below it -- which is why this entry is never refused. Measured on + # 26.7: plain DDL declares the type with no experimental setting, scalar + # path `JSONExtractString`, document path `JSONExtractRaw`, has-key + # `JSONHas`, array length `JSONLength(toString(doc), 'arr')`, validity + # `isValidJSON(toString(doc))`, equality `toString(doc) = ?`. Note the + # `toString` in two of those: passing the JSON column straight to + # `JSONLength` silently returns 0. The version gate that used to hang off + # this entry is now recorded in the module docstring. + dict: JSONColumn, + # --- arrays ---------------------------------------------------------- + # ClickHouse's `Array(T)`, and the one backend family where ArrayColumn + # is the *native* answer rather than a narrowed substitute: the 26.7 + # column of the array matrix is check-marked for every row -- DDL, + # a Python list bound directly (and read back as a list, no framework + # serialisation needed), `length(x)`, 1-based `x[1]`, `has(x, v)`, + # `arrayConcat(x, [v])`, `arrayExists(e -> e = v, x)`, whole-column + # equality `x = [...]`, and `ARRAY JOIN` for unnest. Nothing is + # narrowed on ArrayColumn. + list: ArrayColumn, + # A tuple is heterogeneous and ClickHouse has `Tuple(...)` for exactly + # that, so this cell is the one most likely to move: it is 议题 A's + # subject, and the answer here is the agreed table's, not a ruling that + # `Array(T)` is the right carrier for a heterogeneous value. + tuple: ArrayColumn, + set: ArrayColumn, + # A set is an Array with the duplicates already removed by Python; the + # Array does not re-deduplicate on the server, so "set-ness" is a + # value-layer property and the storage family is `Array(T)`. + frozenset: ArrayColumn, + # Native `Enum8` / `Enum16` storage holding string values, so the value + # semantics are text: comparison, `IN` and `ORDER BY` render. Two + # measured facts a caller must not assume away: `ORDER BY` on a native + # enum is **declaration order**, not lexicographic, and an illegal member + # is refused by the server (code 691) rather than by a CHECK constraint. + enum.Enum: StringColumn, +} + + +class ClickHouseColumnTypeMixin(ColumnTypeMixin): + """ClickHouse's answer to "which column class does this annotation mean". + + Composed into ``ClickHouseDialect`` **ahead of** ``ColumnTypeMixin``, which + is where the backend's own answer wins: it overrides + :meth:`suggested_column_types` and calls ``super()`` for nothing, because + the one other member of the protocol -- ``suggested_extra_column_types`` -- + has no backend-specific answer here. Core offers no inherited table to fall + back on, so the ordering only matters for the method this class defines. + """ + + def suggested_column_types(self) -> Dict[Any, Optional[Type[ColumnBase]]]: + """A fresh copy of :data:`CLICKHOUSE_COLUMN_TYPES`, all eighteen entries. + + Returning a copy is part of the protocol: a caller mutating the answer + must not corrupt the table the next lookup reads. + """ + return dict(CLICKHOUSE_COLUMN_TYPES) + + +__all__ = ["CLICKHOUSE_COLUMN_TYPES", "ClickHouseColumnTypeMixin"] diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/ddl_column.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/ddl_column.py index 988556e..625f289 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/ddl_column.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/ddl_column.py @@ -48,34 +48,59 @@ def format_add_column_action(self, action) -> Tuple[str, tuple]: return " ".join(parts), column_params def format_alter_column_action(self, action) -> Tuple[str, tuple]: - """Format ALTER TABLE ... ALTER COLUMN {SET DEFAULT | DROP DEFAULT}. - - ClickHouse 8.0 syntax is ``ALTER TABLE t ALTER [COLUMN] col {SET DEFAULT - literal | DROP DEFAULT}``. Unlike the generic SQL-standard renderer, - ClickHouse requires a literal for SET DEFAULT (no parenthesised - expressions / parameters), so we inline the value. + """Format ALTER TABLE ... ALTER/MODIFY COLUMN for a default change. + + ClickHouse has no ``SET DEFAULT`` / ``DROP DEFAULT`` action. What it has, + per ``https://clickhouse.com/docs/reference/statements/alter/column`` and + measured on 26.7.3.19: + + ===================================== ========================================== + intent ClickHouse + ===================================== ========================================== + set a default expression ``ALTER COLUMN c [TYPE] DEFAULT expr`` + remove a default expression ``MODIFY COLUMN c REMOVE DEFAULT`` + ===================================== ========================================== + + The MySQL spellings this used to emit are both rejected. Quoting the + server, for ``ALTER TABLE t ALTER COLUMN a SET DEFAULT 5``:: + + Code: 62. DB::Exception: Syntax error: failed at position 45 (SET): + SET DEFAULT 5. Expected one of: token sequence, Dot, token, REMOVE, + MODIFY SETTING, RESET SETTING, ADD ENUM VALUES, NULL, NOT, DEFAULT, + MATERIALIZED, EPHEMERAL, ALIAS, AUTO_INCREMENT, TTL, PRIMARY KEY, + COMMENT, CODEC, TYPE. (SYNTAX_ERROR) + + ``DEFAULT`` is on that list and ``SET`` is not, so the keyword ``SET`` is + the whole difference; and for ``ALTER TABLE t ALTER COLUMN a DROP + DEFAULT`` the same error names no ``DROP`` at all — removing a property + is ``MODIFY COLUMN ... REMOVE``, which the reference page lists for + ``DEFAULT``, ``ALIAS``, ``MATERIALIZED``, ``CODEC``, ``COMMENT``, ``TTL`` + and ``SETTINGS``. + + Note that ClickHouse does require a literal default expression here, so + the value is inlined rather than parameterised. """ operation = getattr(action.operation, "value", None) or str(action.operation) col_name = self.format_identifier(action.column_name) if operation == "DROP DEFAULT": - return f"ALTER COLUMN {col_name} DROP DEFAULT", () + return f"MODIFY COLUMN {col_name} REMOVE DEFAULT", () if operation == "SET DEFAULT": new_value = getattr(action, "new_value", None) if isinstance(new_value, str): escaped = self._escape_sql_string(new_value) - return f"ALTER COLUMN {col_name} SET DEFAULT '{escaped}'", () + return f"ALTER COLUMN {col_name} DEFAULT '{escaped}'", () if isinstance(new_value, bool): - return f"ALTER COLUMN {col_name} SET DEFAULT {1 if new_value else 0}", () + return f"ALTER COLUMN {col_name} DEFAULT {1 if new_value else 0}", () if new_value is None: raise ValueError("SET DEFAULT requires a default value") if isinstance(new_value, (int, float)): - return f"ALTER COLUMN {col_name} SET DEFAULT {new_value}", () + return f"ALTER COLUMN {col_name} DEFAULT {new_value}", () if hasattr(new_value, "to_sql"): value_sql, value_params = new_value.to_sql() - return f"ALTER COLUMN {col_name} SET DEFAULT {value_sql}", tuple(value_params) - return f"ALTER COLUMN {col_name} SET DEFAULT {new_value}", () + return f"ALTER COLUMN {col_name} DEFAULT {value_sql}", tuple(value_params) + return f"ALTER COLUMN {col_name} DEFAULT {new_value}", () return super().format_alter_column_action(action) @@ -85,8 +110,36 @@ def format_table_constraint( ) -> Tuple[str, tuple]: """Format a table-level constraint. - ClickHouse supports only PRIMARY KEY among table-level constraints; - UNIQUE and FOREIGN KEY constraints are not supported. + ClickHouse has two table-level constraint forms and they are not these: + ``CONSTRAINT name CHECK `` and ``CONSTRAINT name ASSUME `` + (https://clickhouse.com/docs/reference/statements/create/table#constraints). + Only ``PRIMARY KEY`` is rendered from the SQL-standard set. + + ``UNIQUE`` and ``FOREIGN KEY`` refuse rather than emit, and the reason is + not symmetric: + + * ``UNIQUE (col)`` is a ``SYNTAX_ERROR`` on 26.7.3.19 — ``Code: 62. + Syntax error: failed at position 50 ((): (col)). Expected one of: NULL, + NOT, DEFAULT, MATERIALIZED, EPHEMERAL, ALIAS, AUTO_INCREMENT, TTL, + PRIMARY KEY, data type, identifier. (SYNTAX_ERROR)`` — so rendering it + would be an error the caller sees immediately. + * ``FOREIGN KEY`` **without** a ``CONSTRAINT`` name is *accepted and + silently discarded*, and that is why it refuses. Measured on + 26.7.3.19: ``CREATE TABLE c (id UInt32, pid UInt32, FOREIGN KEY (pid) + REFERENCES p(id)) ENGINE = MergeTree ORDER BY id`` succeeds, but + ``SHOW CREATE TABLE c`` reports no foreign key, + ``system.tables.create_table_query`` does not contain the string + ``FOREIGN``, an orphan row inserts and reads back fine, and dropping + the referenced parent also succeeds. (The *named* form ``CONSTRAINT fk + FOREIGN KEY ...`` is a ``SYNTAX_ERROR`` expecting ``CHECK`` or + ``ASSUME``, as are the inline column-level ``REFERENCES p(id)`` and + the form written after the storage clauses.) + + So emitting it would be the worst of the three outcomes: no error at + DDL time, no record in the schema, and no enforcement at write time. + The referential-integrity bug would surface in production data with + nothing in the server's metadata to explain it. Refusing turns that + into an ``UnsupportedFeatureError`` at the point of declaration. """ from rhosocial.activerecord.backend.expression.statements.ddl_table import ( TableConstraintType, @@ -106,7 +159,13 @@ def format_table_constraint( elif t_const.constraint_type == TableConstraintType.FOREIGN_KEY: raise UnsupportedFeatureError( self.name, "FOREIGN KEY constraint", - suggestion="ClickHouse does not support FOREIGN KEY constraints." + suggestion=( + "ClickHouse does not support FOREIGN KEY constraints, and an " + "unnamed FOREIGN KEY (...) REFERENCES ... clause is accepted and " + "then silently discarded, so emitting one would produce a schema " + "with no foreign key and no enforcement. Maintain referential " + "integrity in the application, or use CONSTRAINT ... CHECK." + ) ) return ' '.join(parts), tuple(params) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/fulltext.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/fulltext.py index 6044ea8..f5078c4 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/fulltext.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/fulltext.py @@ -9,8 +9,9 @@ class ClickHouseFullTextSearchMixin: ClickHouse does not support standard MySQL-style FULLTEXT indexes or MATCH ... AGAINST search. All supports_* methods return False and - format methods raise UnsupportedFeatureError. ClickHouse uses skip - indexes (INDEX ... USING) instead of FULLTEXT. + format methods raise UnsupportedFeatureError. ClickHouse's own facility is + the ``text`` inverted index (GA from 26.2), declared as a skip index and + queried with ``hasAllTokens`` / ``hasAnyTokens``. """ def supports_fulltext_index(self) -> bool: @@ -30,11 +31,14 @@ def format_fulltext_index_options( ) -> Tuple[str, tuple]: raise UnsupportedFeatureError( self.name, "FULLTEXT index", - suggestion="ClickHouse does not support FULLTEXT indexes; use skip indexes (INDEX ... USING)." + suggestion="ClickHouse does not support FULLTEXT indexes; use a text inverted index " + "skip index (INDEX ... TYPE text(tokenizer = splitByNonAlpha)) queried with " + "hasAllTokens / hasAnyTokens. tokenbf_v1 still works but is deprecated from 26.2.", ) def format_match_against(self, expr) -> Tuple[str, tuple]: raise UnsupportedFeatureError( self.name, "MATCH ... AGAINST", - suggestion="ClickHouse does not support MATCH ... AGAINST; use LIKE, hasToken, or tokenbf_v1 skip indexes." + suggestion="ClickHouse does not support MATCH ... AGAINST; use a text inverted index " + "with hasAllTokens / hasAnyTokens, or LIKE.", ) \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/index.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/index.py index 21a15a2..10a6ebc 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/index.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/index.py @@ -44,5 +44,11 @@ def supports_functional_index(self) -> bool: return True def supports_index_type(self) -> bool: - """ClickHouse skip indexes support USING keyword for index type.""" + """Whether index type specification is supported. + + ClickHouse skip indexes carry a type, but the keyword is ``TYPE``, not + MySQL's ``USING``: ``INDEX i (s) TYPE minmax GRANULARITY 1`` is accepted + on 26.7.3.19, while ``INDEX i (s) USING minmax`` is a ``SYNTAX_ERROR`` + ("failed at position 54 (USING)"). + """ return True diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/introspection.py index 45b4b77..3c10e85 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/introspection.py @@ -37,12 +37,15 @@ class ClickHouseIntrospectionMixin: trigger introspection are NOT supported (ClickHouse has neither FKs nor triggers), so those supports_* methods return False. - NOTE: The format_*_query methods below currently generate MySQL-style - information_schema SQL inherited from the MySQL backend. They are known - MySQL remnants that still need to be rewritten against ClickHouse system - tables (system.databases / system.tables / system.columns / system.views - / system.data_skipping_indices). index/foreign_key/trigger queries have - no direct system-table equivalent in ClickHouse. + NOTE: Every ``format_*_query`` below already reads a ClickHouse system + table — ``system.databases``, ``system.tables``, ``system.columns``, + ``system.data_skipping_indices``, and the ``engine LIKE '%View'`` subset of + ``system.tables``. (An earlier note here claimed these still emitted MySQL + ``information_schema`` SQL "inherited from the MySQL backend"; that was + stale, they had already been rewritten.) Column aliases such as + ``AS TABLE_NAME`` are kept because the result parsers upstream read those + names; the *sources* are ClickHouse's own. Foreign-key and trigger queries + have no ClickHouse system table at all and raise. """ # ========== Capability Detection ========== diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/json.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/json.py index ac45038..99cefcf 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/json.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/json.py @@ -1,10 +1,20 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/mixins/json.py -from typing import Any, List, Tuple, TYPE_CHECKING +from typing import Union, Any, List, Tuple, TYPE_CHECKING from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError if TYPE_CHECKING: - from rhosocial.activerecord.backend.expression.advanced_functions import JSONExpression + from rhosocial.activerecord.backend.expression.advanced_functions import ( + JSONDocumentExpression, + JSONTextExpression, + ) + + #: Core has one class per JSON operator, and this dialect renders either + #: from the same code path -- so the parameter is a union, not one of the + #: two. ``JSONTextExpression`` subclasses ``JSONDocumentExpression`` so a + #: path chain can keep going after ``->>``; that inheritance is about + #: accessors, not about ``->>`` yielding a document. + JSONPathNode = Union[JSONDocumentExpression, JSONTextExpression] class ClickHouseJSONFunctionMixin: @@ -32,7 +42,7 @@ def supports_json_arrow_operators(self) -> bool: """ClickHouse does not support the MySQL-style ``->`` / ``->>`` operators.""" return False - def format_json_function_expression(self, expr: "JSONExpression") -> Tuple[str, tuple]: + def format_json_function_expression(self, expr: "JSONPathNode") -> Tuple[str, tuple]: """Format a JSON path expression using native ClickHouse functions. ``->`` (JSON value) maps to ``JSONExtractRaw(col, ...parts...)`` diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/partition.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/partition.py index 7a8a60a..9690d7e 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/partition.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/partition.py @@ -1,302 +1,234 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/mixins/partition.py -from typing import Sequence, Tuple, Union, TYPE_CHECKING +"""ClickHouse table-partition capability and rendering. + +ClickHouse partitioning is a property of the ``MergeTree`` family: a +``PARTITION BY `` clause on ``CREATE TABLE``, where the expression is +any expression over the table's columns (or a tuple of them). There is no +strategy to pick, no ``VALUES`` boundary and no subpartitioning. Declaring the +clause is done by the storage-clause formatter +(:meth:`~.ddl_table_engine.ClickHouseTableEngineMixin.format_table_engine_clauses`, +which renders ``PARTITION BY`` out of the ``storage_options`` mapping); what is +left for this mixin is the capability switches and the three ``ALTER TABLE`` +maintenance clauses ClickHouse actually has, each addressed by **partition id**. + +References (both fetched and HTTP-resolving): +https://clickhouse.com/docs/engines/table-engines/mergetree-family/custom-partitioning-key +https://clickhouse.com/docs/sql-reference/statements/alter/partition +""" +from typing import NoReturn, Tuple, TYPE_CHECKING from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError if TYPE_CHECKING: # pragma: no cover - from rhosocial.activerecord.backend.expression.statements import PartitionClause - from rhosocial.activerecord.backend.impl.clickhouse.expression.partition import ( - ClickHouseAddPartitionExpression, + from ..expression.partition import ( + ClickHouseAttachPartitionExpression, + ClickHouseDetachPartitionExpression, ClickHouseDropPartitionExpression, - ClickHouseGetPartitionsExpression, - ClickHousePartitionByHash, - ClickHousePartitionByKey, - ClickHousePartitionByList, - ClickHousePartitionByListColumns, - ClickHousePartitionByRange, - ClickHousePartitionByRangeColumns, - ClickHousePartitionDefinition, - ClickHousePartitionMaxValue, - ClickHousePartitionNameListExpression, - ClickHousePartitionValue, - ClickHouseExchangePartitionExpression, - ClickHouseReorganizePartitionExpression, - ClickHouseTruncatePartitionExpression, - ClickHouseRemovePartitioningExpression, - ClickHouseCoalescePartitionExpression, - ClickHouseAnalyzePartitionExpression, - ClickHouseCheckPartitionExpression, - ClickHouseOptimizePartitionExpression, - ClickHouseRebuildPartitionExpression, - ClickHouseRepairPartitionExpression, - ClickHouseSubpartitionClause, - ClickHouseSubpartitionDefinition, ) +#: The ``ALTER TABLE ... PARTITION`` inventory, transcribed from the ALTER TABLE +#: ... PARTITION reference cited below. This is also what proves the absence of +#: everything else, so it is kept as data rather than prose: a test asserts +#: that nothing in :data:`MYSQL_ONLY_PARTITION_STATEMENTS` / +#: :data:`MYSQL_ONLY_PARTITION_SYNTAX` appears in it. +#: +#: The server's own expectation list is a mild superset of this tuple — it also +#: offers partition-level ``MODIFY TTL`` / ``REMOVE TTL`` / ``MATERIALIZE TTL`` +#: and ``UNLOCK SNAPSHOT`` — so nothing here is wrong, but the tuple is not the +#: whole of what ClickHouse accepts either. +ALTER_PARTITION_INVENTORY = ( + "DETACH PARTITION|PART", + "DROP PARTITION|PART", + "DROP DETACHED PARTITION|PART", + "FORGET PARTITION", + "ATTACH PARTITION|PART", + "ATTACH PARTITION FROM", + "REPLACE PARTITION", + "MOVE PARTITION TO TABLE", + "CLEAR COLUMN IN PARTITION", + "CLEAR INDEX IN PARTITION", + "FREEZE PARTITION", + "UNFREEZE PARTITION", + "FETCH PARTITION|PART", + "MOVE PARTITION|PART TO DISK|VOLUME", + "UPDATE IN PARTITION", + "DELETE IN PARTITION", + "REWRITE PARTS", +) + +#: MySQL ``ALTER TABLE`` partition statements with no ClickHouse spelling. +#: Nothing in :data:`ALTER_PARTITION_INVENTORY` matches any of these; asking the +#: server for one returns its expectation list naming every clause it does +#: accept, and none of those names is among them. +MYSQL_ONLY_PARTITION_STATEMENTS = ( + "ADD PARTITION", + "TRUNCATE PARTITION", + "REORGANIZE PARTITION", + "EXCHANGE PARTITION", + "REMOVE PARTITIONING", + "COALESCE PARTITION", + "ANALYZE PARTITION", + "CHECK PARTITION", + "OPTIMIZE PARTITION", + "REBUILD PARTITION", + "REPAIR PARTITION", + "SUBPARTITION BY", +) + +#: MySQL inline partition syntax with no ClickHouse spelling. Not a statement +#: name, so kept apart from :data:`MYSQL_ONLY_PARTITION_STATEMENTS` to keep that +#: tuple comparable against :data:`ALTER_PARTITION_INVENTORY` entry by entry. +MYSQL_ONLY_PARTITION_SYNTAX = ( + "PARTITION ... VALUES LESS THAN", + "PARTITION ... VALUES IN", + "MAXVALUE partition boundary", +) + +_PARTITION_ALTERNATIVES = ( + "ClickHouse partitions a MergeTree table with PARTITION BY in " + "CREATE TABLE, not with a named RANGE/LIST/HASH/KEY strategy; it is " + "declared through the storage_options mapping. A partition comes into " + "existence when a row lands in it, so there is nothing to ADD, and its " + "contents are addressed by system.parts.partition_id rather than by a " + "DDL-declared name." +) + + class ClickHousePartitionMixin: - """ClickHouse table partitioning implementation. - - WARNING: ClickHouse does not support MySQL declarative partitioning - (``PARTITION BY RANGE/LIST/HASH/KEY`` with explicit partition definitions, - subpartitioning, and ``ALTER TABLE ... PARTITION`` maintenance statements). - ClickHouse partitioning is expressed as a ``PARTITION BY`` expression in - ``CREATE TABLE``, handled by ``ClickHouseTableEngineMixin``. All MySQL - declarative partition SQL generation methods in this class raise - ``UnsupportedFeatureError``; the corresponding ``supports_*`` switches - return ``False``. + """ClickHouse table-partition support. + + Covers the two things ClickHouse genuinely has — the ``PARTITION BY`` + capability and partition maintenance by partition id — and refuses + everything the MySQL partition surface asked for. The refusals are + overrides of the generic ``PartitionSupport`` contract rather than one + ``supports_*`` switch per MySQL statement: a switch per MySQL statement + would be an inventory of statements ClickHouse does not have, and the + authoritative inventory is + :data:`ALTER_PARTITION_INVENTORY` (see :mod:`..expression.partition`). """ + # ------------------------------------------------------------------ + # Capabilities + # ------------------------------------------------------------------ + def supports_table_partitioning(self) -> bool: - """ClickHouse supports ``PARTITION BY`` expression in CREATE TABLE.""" + """ClickHouse partitions tables, via ``PARTITION BY `` on MergeTree.""" return True def supports_partitioned_table_creation(self) -> bool: - """ClickHouse supports ``PARTITION BY`` in partitioned table creation.""" + """``CREATE TABLE`` may carry the ``PARTITION BY`` clause.""" return True - def supports_range_table_partitioning(self) -> bool: - return False - - def supports_list_table_partitioning(self) -> bool: - return False - - def supports_hash_table_partitioning(self) -> bool: - return False - - def supports_key_table_partitioning(self) -> bool: - return False - - def supports_subpartitioning(self) -> bool: - """ClickHouse has no subpartitioning.""" - return False - - def supports_range_columns_partitioning(self) -> bool: - return False - - def supports_list_columns_partitioning(self) -> bool: - return False - - def supports_linear_hash_partitioning(self) -> bool: - return False - - def supports_linear_key_partitioning(self) -> bool: - return False + def supports_partition_metadata_introspection(self) -> bool: + """ClickHouse reports partitions in ``system.parts``. - def supports_add_partition(self) -> bool: - return False + The columns that carry the partition are ``partition`` (the key value), + ``partition_id`` (the id ``ALTER TABLE ... PARTITION ID`` addresses) and + ``name`` (the part name). ``information_schema.PARTITIONS`` is MySQL's + and the server rejects that table name. + """ + return True def supports_drop_partition(self) -> bool: - return False - - def supports_truncate_partition(self) -> bool: - return False - - def supports_reorganize_partition(self) -> bool: - return False - - def supports_attach_partition(self) -> bool: - return False + """``ALTER TABLE ... DROP PARTITION ID`` — supported.""" + return True def supports_detach_partition(self) -> bool: - return False - - def supports_partition_metadata_introspection(self) -> bool: - """ClickHouse partition introspection uses ``system.parts``, not - MySQL ``information_schema.PARTITIONS``.""" - return False - - def supports_partition_definition_options(self) -> bool: - return False - - def supports_partition_value_maxvalue(self) -> bool: - return False - - def supports_remove_partitioning(self) -> bool: - return False - - def supports_coalesce_partition(self) -> bool: - return False - - def supports_exchange_partition(self) -> bool: - return False - - def supports_analyze_partition(self) -> bool: - return False - - def supports_check_partition(self) -> bool: - return False - - def supports_optimize_partition(self) -> bool: - return False + """``ALTER TABLE ... DETACH PARTITION ID`` — supported.""" + return True - def supports_rebuild_partition(self) -> bool: - return False + def supports_attach_partition(self) -> bool: + """``ALTER TABLE ... ATTACH PARTITION ID`` — supported.""" + return True - def supports_repair_partition(self) -> bool: - return False + # ------------------------------------------------------------------ + # Refusals + # ------------------------------------------------------------------ - def _unsupported(self, feature: str) -> None: - """Raise UnsupportedFeatureError for MySQL declarative partitioning. + def format_partition_clause(self, expr) -> Tuple[str, tuple]: + """Refuse a generic ``PartitionClause``. - ClickHouse partitioning is a ``PARTITION BY`` expression in - ``CREATE TABLE`` (see ``ClickHouseTableEngineMixin``), not MySQL - declarative partitioning. + A generic ``PartitionClause`` can only be spelled ``RANGE``/``LIST``/ + ``HASH`` (or the backend's own enum), and none of those is a ClickHouse + partition key — ClickHouse takes an arbitrary expression instead. The + honest refusal points at where ``PARTITION BY`` *is* written. """ - raise UnsupportedFeatureError( - self.name, - feature, - suggestion="ClickHouse uses PARTITION BY expression in CREATE TABLE, not MySQL declarative partitioning.", - ) + self._refuse_mysql_partition("declarative PARTITION BY ") - def format_partition_clause(self, expr: "PartitionClause") -> Tuple[str, tuple]: - """Format ClickHouse PARTITION BY clause from a PartitionClause expression. + def format_partition_definition(self, definition) -> Tuple[str, tuple]: + """Refuse an inline ``PARTITION ... VALUES ...`` definition. - MySQL declarative partitioning is not supported by ClickHouse. + ClickHouse has no inline partition definitions: a partition is not + declared in the table's DDL, so there is nothing for a definition to + render. """ - self._unsupported(f"{expr.method} declarative partitioning") - - def format_partition_by_range(self, expr: "ClickHousePartitionByRange") -> Tuple[str, tuple]: - """MySQL ``PARTITION BY RANGE`` is not supported by ClickHouse.""" - self._unsupported("RANGE declarative partitioning") + self._refuse_mysql_partition("inline partition definitions") - def format_partition_by_range_columns(self, expr: "ClickHousePartitionByRangeColumns") -> Tuple[str, tuple]: - """MySQL ``PARTITION BY RANGE COLUMNS`` is not supported by ClickHouse.""" - self._unsupported("RANGE COLUMNS declarative partitioning") + def _refuse_mysql_partition(self, feature: str) -> NoReturn: + """Raise ``UnsupportedFeatureError`` naming ClickHouse's partitioning.""" + raise UnsupportedFeatureError(self.name, feature, _PARTITION_ALTERNATIVES) - def format_partition_by_list(self, expr: "ClickHousePartitionByList") -> Tuple[str, tuple]: - """MySQL ``PARTITION BY LIST`` is not supported by ClickHouse.""" - self._unsupported("LIST declarative partitioning") + # ------------------------------------------------------------------ + # Formatters — the three maintenance clauses ClickHouse has + # ------------------------------------------------------------------ - def format_partition_by_list_columns(self, expr: "ClickHousePartitionByListColumns") -> Tuple[str, tuple]: - """MySQL ``PARTITION BY LIST COLUMNS`` is not supported by ClickHouse.""" - self._unsupported("LIST COLUMNS declarative partitioning") - - def format_partition_by_hash(self, expr: "ClickHousePartitionByHash") -> Tuple[str, tuple]: - """MySQL ``PARTITION BY HASH`` is not supported by ClickHouse.""" - self._unsupported("HASH declarative partitioning") - - def format_partition_by_key(self, expr: "ClickHousePartitionByKey") -> Tuple[str, tuple]: - """MySQL ``PARTITION BY KEY`` is not supported by ClickHouse.""" - self._unsupported("KEY declarative partitioning") - - def format_partition_definition(self, definition: "ClickHousePartitionDefinition") -> Tuple[str, tuple]: - """MySQL partition definitions are not supported by ClickHouse.""" - self._unsupported("partition definitions") - - def format_partition_definition_options(self, options: dict) -> Tuple[str, tuple]: - """MySQL partition definition options are not supported by ClickHouse.""" - self._unsupported("partition definition options") - - def format_get_partitions_expression(self, expr: "ClickHouseGetPartitionsExpression") -> Tuple[str, tuple]: - """MySQL ``SELECT ... FROM information_schema.PARTITIONS`` is not supported. - - ClickHouse partition introspection uses the ``system.parts`` table. - - Args: - expr: ClickHouseGetPartitionsExpression with the target table name. + def format_drop_partition_statement( + self, expr: "ClickHouseDropPartitionExpression" + ) -> Tuple[str, tuple]: + """Render ``ALTER TABLE ... DROP PARTITION ID ''``. Raises: - UnsupportedFeatureError: always. + UnsupportedFeatureError: if ``expr`` does not carry a partition id. """ - self._unsupported("information_schema.PARTITIONS introspection") + return self._format_partition_id_statement(expr, "DROP") - def format_partition_value( - self, - expr: Union["ClickHousePartitionValue", "ClickHousePartitionMaxValue"], + def format_detach_partition_statement( + self, expr: "ClickHouseDetachPartitionExpression" ) -> Tuple[str, tuple]: - """MySQL partition boundary values are not supported by ClickHouse.""" - self._unsupported("partition boundary VALUES") - - def format_subpartition_by(self, expr: "ClickHouseSubpartitionClause") -> Tuple[str, tuple]: - """MySQL ``SUBPARTITION BY`` is not supported by ClickHouse.""" - self._unsupported("subpartitioning") + """Render ``ALTER TABLE ... DETACH PARTITION ID ''``. - def format_subpartition_definition(self, definition: "ClickHouseSubpartitionDefinition") -> Tuple[str, tuple]: - """MySQL subpartition definitions are not supported by ClickHouse.""" - self._unsupported("subpartition definitions") - - def format_add_partition_statement(self, expr: "ClickHouseAddPartitionExpression") -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... ADD PARTITION`` is not supported by ClickHouse.""" - self._unsupported("ADD PARTITION") - - def format_drop_partition_statement(self, expr: "ClickHouseDropPartitionExpression") -> Tuple[str, tuple]: - """MySQL declarative ``ALTER TABLE ... DROP PARTITION`` is not supported. - - ClickHouse removes partitions by partition-id via - ``ALTER TABLE ... DROP PARTITION``. + Raises: + UnsupportedFeatureError: if ``expr`` does not carry a partition id. """ - self._unsupported("declarative DROP PARTITION") + return self._format_partition_id_statement(expr, "DETACH") - def format_truncate_partition_statement(self, expr: "ClickHouseTruncatePartitionExpression") -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... TRUNCATE PARTITION`` is not supported by ClickHouse.""" - self._unsupported("TRUNCATE PARTITION") - - def format_reorganize_partition_statement( - self, - expr: "ClickHouseReorganizePartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... REORGANIZE PARTITION`` is not supported by ClickHouse.""" - self._unsupported("REORGANIZE PARTITION") - - def format_exchange_partition_statement( - self, - expr: "ClickHouseExchangePartitionExpression", + def format_attach_partition_statement( + self, expr: "ClickHouseAttachPartitionExpression" ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... EXCHANGE PARTITION`` is not supported by ClickHouse.""" - self._unsupported("EXCHANGE PARTITION") + """Render ``ALTER TABLE ... ATTACH PARTITION ID ''``. - def format_partition_name_list( - self, expr: "ClickHousePartitionNameListExpression" - ) -> Tuple[str, tuple]: - """MySQL partition-name lists are not supported by ClickHouse.""" - self._unsupported("partition name list") - - def format_remove_partitioning_statement( - self, - expr: "ClickHouseRemovePartitioningExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... REMOVE PARTITIONING`` is not supported by ClickHouse.""" - self._unsupported("REMOVE PARTITIONING") - - def format_coalesce_partition_statement( - self, - expr: "ClickHouseCoalescePartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... COALESCE PARTITION`` is not supported by ClickHouse.""" - self._unsupported("COALESCE PARTITION") - - def format_analyze_partition_statement( - self, - expr: "ClickHouseAnalyzePartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... ANALYZE PARTITION`` is not supported by ClickHouse.""" - self._unsupported("ANALYZE PARTITION") - - def format_check_partition_statement( - self, - expr: "ClickHouseCheckPartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... CHECK PARTITION`` is not supported by ClickHouse.""" - self._unsupported("CHECK PARTITION") - - def format_optimize_partition_statement( - self, - expr: "ClickHouseOptimizePartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... OPTIMIZE PARTITION`` is not supported by ClickHouse.""" - self._unsupported("OPTIMIZE PARTITION") + Raises: + UnsupportedFeatureError: if ``expr`` does not carry a partition id. + """ + return self._format_partition_id_statement(expr, "ATTACH") - def format_rebuild_partition_statement( - self, - expr: "ClickHouseRebuildPartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... REBUILD PARTITION`` is not supported by ClickHouse.""" - self._unsupported("REBUILD PARTITION") + def _format_partition_id_statement(self, expr, verb: str) -> Tuple[str, tuple]: + """Render one ``ALTER TABLE ... PARTITION ID ''`` statement. - def format_repair_partition_statement( - self, - expr: "ClickHouseRepairPartitionExpression", - ) -> Tuple[str, tuple]: - """MySQL ``ALTER TABLE ... REPAIR PARTITION`` is not supported by ClickHouse.""" - self._unsupported("REPAIR PARTITION") + The id is an inline literal rather than a bind parameter because an + ``ALTER`` clause is DDL-shaped: the partition id is chosen by the + caller, not derived from a column value, so there is no query-planning + benefit in parameterising it. + """ + partition_id = getattr(expr, "partition_id", None) + if not isinstance(partition_id, str) or not partition_id.strip(): + raise UnsupportedFeatureError( + self.name, + f"{verb} PARTITION", + _PARTITION_ALTERNATIVES, + ) + from rhosocial.activerecord.backend.expression.objects import Table + + # The expression stores the bare name (and the database, when given) + # so its round trip carries exactly what the constructor accepted; the + # catalog slot is where ClickHouse keeps its database, so that is where + # the name is qualified here, at render time. + table_sql, table_params = self.format_table_object( + Table(self, expr.table, catalog_name=expr.schema) + ) + return ( + f"ALTER TABLE {table_sql} {verb} PARTITION ID " + f"{self.format_literal(partition_id)}", + tuple(table_params), + ) \ No newline at end of file diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/spatial.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/spatial.py index 9fc8e04..e0803ba 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/spatial.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/spatial.py @@ -88,5 +88,6 @@ def format_st_contains(self, expr) -> Tuple[str, tuple]: def format_create_spatial_index(self, expr) -> Tuple[str, tuple]: raise UnsupportedFeatureError( self.name, "SPATIAL indexes", - suggestion="ClickHouse has no SPATIAL indexes; use skip indexes (INDEX ... USING).", + suggestion="ClickHouse has no SPATIAL indexes; use a skip index declared as " + "INDEX name (col) TYPE GRANULARITY n (e.g. minmax, set, bloom_filter, text).", ) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/trim.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/trim.py new file mode 100644 index 0000000..af4923d --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/trim.py @@ -0,0 +1,67 @@ +# src/rhosocial/activerecord/backend/impl/clickhouse/mixins/trim.py +"""ClickHouse's TRIM spelling: the direction is the function name. + +ClickHouse parses the ANSI ``TRIM([LEADING|TRAILING|BOTH] [chars] FROM s)`` +form too, but only when the character set is spelled out -- +``TRIM(BOTH FROM s)`` (no characters) is a syntax error measured on the public +playground: ``Syntax error: failed at position ... Expected FROM``. The shared +default renderer emits exactly that form for the whitespace case, so it is not +safe here. ClickHouse's own function form takes the characters optionally: + + BOTH -> ``trimBoth(x[, chars])`` (alias ``trim``) + LEADING -> ``trimLeft(x[, chars])`` (alias ``ltrim``) + TRAILING -> ``trimRight(x[, chars])`` (alias ``rtrim``) + +so the mapping is a substitution of the function name plus the operand order +(the target first, the optional character set second). The three names carry +the direction the ANSI keyword carries, which is why no direction operand +survives into the rendered SQL. Everything else about the node -- the operand +parameters in order and the optional alias -- is the default's. + +The functions arrived in v20.1.0 (``trimBoth`` / ``trimLeft`` / ``trimRight``); +earlier releases need the ANSI form, and this mixin does not attempt to gate on +the server version because the dialect has no mechanism for it. +""" +from typing import Tuple, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from ...expression.advanced_functions import TrimExpression + + +class ClickHouseTrimMixin: + """Mixin rendering the TRIM family as ClickHouse's trimBoth/Left/Right.""" + + _TRIM_FUNCTIONS = {"BOTH": "trimBoth", "LEADING": "trimLeft", "TRAILING": "trimRight"} + + def format_trim_expression(self, expr: "TrimExpression") -> Tuple[str, Tuple]: + """Format a TRIM node in ClickHouse's form. + + Args: + expr: Trim expression exposing ``expr``, ``chars``, ``direction`` + and an optional ``alias``. + + Returns: + Tuple of (SQL string, parameters tuple), carrying the operands' + parameters in order. + + Raises: + ValueError: ``expr.direction`` is not one of the three the node + allows. + """ + target_sql, target_params = expr.expr.to_sql() + function = self._TRIM_FUNCTIONS.get(expr.direction) + if function is None: + raise ValueError( + f"Unsupported trim direction {expr.direction!r}; expected one of " + f"{sorted(self._TRIM_FUNCTIONS)}" + ) + if expr.chars is not None: + chars_sql, chars_params = expr.chars.to_sql() + sql = f"{function}({target_sql}, {chars_sql})" + params = tuple(target_params) + tuple(chars_params) + else: + sql = f"{function}({target_sql})" + params = tuple(target_params) + if expr.alias: + sql = f"{sql} AS {self.format_identifier(expr.alias)}" + return sql, params diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/types.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/types.py index 30f9750..96b89ae 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/types.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/types.py @@ -6,6 +6,7 @@ import re from typing import Dict, Tuple +from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError from rhosocial.activerecord.backend.dialect.mixins.data_type import DataTypeMixin from rhosocial.activerecord.backend.dialect.protocols import DataTypeSupport from rhosocial.activerecord.backend.expression.types import ( @@ -28,6 +29,7 @@ TimeType, TimestampType, TinyIntType, + UUIDType, VarCharType, ) from ..expression.types import ( @@ -90,34 +92,260 @@ class ClickHouseTypeSupportMixin(DataTypeMixin, DataTypeSupport): names; core types render their real ClickHouse SQL. """ + #: First server version on which ClickHouse's ``QBit`` vector type needs no + #: experimental opt-in. ``QBit`` appeared in 25.10 behind + #: ``allow_experimental_qbit_type`` and was promoted to General Availability in + #: 26.2; on 26.7 that setting already defaults to ``1``. So this is the GA + #: version, and it is what :meth:`format_data_type_clickhouse_vector` gates on. + _QBIT_MIN_VERSION = (26, 2, 0) + # ------------------------------------------------------------------ # DataTypeSupport — formatting # ------------------------------------------------------------------ # --- ClickHouse-specific type formatters (dispatch key = type name) --- + def _refuse_unsigned_integer( + self, + data_type, + signed_word: str, + unsigned_class: str, + ) -> None: + """Refuse ``unsigned=True`` on a *signed* ClickHouse integer. + + ``unsigned`` is in the integer concepts' ``PARAMETERS``, so it is part of + their identity: ``ClickHouseInt8Type(unsigned=True)`` is a different + column from ``ClickHouseInt8Type()``, and the differ compares them as such. + A dialect may therefore honour the field — the rendered SQL changes when it + flips — or refuse it, naming the field and why. There is no third option, + and in particular no "accept it and write the same SQL either way": that + would create a column that does not match the declaration and report + success. + + ClickHouse *does* have unsigned integers, so honouring it looks available. + It is not, and the reason is that ClickHouse spells signedness in the + **name**, not as a modifier on the name: + + * "ClickHouse offers a number of fixed-length integers, with a sign + (``Int``) or without a sign (unsigned ``UInt``) ranging from one byte to + 32 bytes." The signed and unsigned forms are two families of type names + over the same eight storage sizes, with disjoint ranges — ``Int8`` is + [-128 : 127] and ``UInt8`` is [0 : 255], ``Int16``/``UInt16``, + ``Int32``/``UInt32``, ``Int64``/``UInt64`` likewise. + https://clickhouse.com/docs/sql-reference/data-types/int-uint + * ``UNSIGNED`` is not an attribute a column declaration can carry after the + native name: ``CREATE TABLE t (a Int8 UNSIGNED)`` fails on 26.7.3.19 with + ``Unknown data type family: INT8 UNSIGNED``. Nothing to append. + * The same chapter lists *aliases*, and the unsigned ones — + ``TINYINT UNSIGNED`` / ``INT1 UNSIGNED``, ``SMALLINT UNSIGNED`` / + ``YEAR``, ``MEDIUMINT UNSIGNED`` / ``INT UNSIGNED`` / + ``INTEGER UNSIGNED``, ``UNSIGNED`` / ``BIGINT UNSIGNED`` — resolve to + ``UInt8``…``UInt64``. Verified on 26.7.3.19: + ``SELECT toTypeName(CAST(1 AS TINYINT UNSIGNED))`` returns ``UInt8``. + So rendering one of those aliases *would* honour the flag — and that is + exactly why honouring is refused. + + **Why the alias is not the answer.** Writing ``TINYINT UNSIGNED`` for + ``ClickHouseInt8Type(unsigned=True)`` produces a column byte-for-byte + identical to the one :class:`ClickHouseUInt8Type` renders, because both are + ``UInt8``. Two declarations for one column is the thing one-concept-one-class + forbids: the caller would have two ways to name the same storage with no way + to tell them apart except by reading a docstring, and a differ comparing + ``ClickHouseUInt8Type()`` against ``ClickHouseInt8Type(unsigned=True)`` + would disagree about whether a column changed. The class already exists for + this column and pins ``unsigned=True``; the flag on the signed class has + nowhere honest to go. + + So all eight formatters that can receive the flag — the four + ``clickhouse_int*`` and the four core ``tinyint``/``smallint``/``integer`` / + ``bigint`` names, which are the same concepts reached by their own + dispatch keys — refuse it the same way and point at the class that does own + the unsigned column. + """ + if not data_type.unsigned: + return + raise UnsupportedFeatureError( + self.name, + f"unsigned=True on {signed_word} " + f"(unsigned is not a modifier on a ClickHouse type name; " + f"ClickHouse spells the unsigned form of {signed_word} as a different " + f"type, and {signed_word} itself has no unsigned variant)", + suggestion=( + f"Use {unsigned_class}, which declares unsigned=True itself and " + f"renders the ClickHouse unsigned integer of that width." + ), + ) + + def _refuse_unsigned_numeric( + self, + data_type, + ch_word: str, + ) -> None: + """Refuse ``unsigned=True`` on a ClickHouse float or decimal. + + ``unsigned`` reaches the four numeric concepts here that it reaches the + integer widths — ``DecimalType``, ``FloatType``, ``DoubleType`` and + ``RealType`` each carry it in ``PARAMETERS``, so two declarations + differing only in it are different columns as far as the differ is + concerned — and ClickHouse's answer is a refusal for a reason the + integers do not share: **ClickHouse spells signedness in the type + *name*, and for floats and decimals it does not spell it at all.** + + Every part of that was measured on the wired scenario server, 26.7.3.19. + + * **The server's own type inventory has no unsigned float or decimal.** + ``system.data_type_families`` names 140 families. The eight whose + names contain ``UNSIGNED`` are all **integer** aliases and every one + of them resolves to a ``UInt`` family:: + + TINYINT UNSIGNED / INT1 UNSIGNED -> UInt8 + SMALLINT UNSIGNED -> UInt16 + MEDIUMINT UNSIGNED / INT UNSIGNED / INTEGER UNSIGNED -> UInt32 + BIGINT UNSIGNED / UNSIGNED -> UInt64 + + There is no ``Decimal ... UNSIGNED``, no ``Float32 UNSIGNED`` and no + ``Float64 UNSIGNED`` among them. The float and decimal families' + own ``syntax`` strings are the whole of their grammar and carry no + attribute at all: ``Float32``, ``Float64``, ``Decimal(P, S)``, + ``Decimal32(S)``, ``Decimal64(S)``, ``Decimal128(S)``, + ``Decimal256(S)``, ``BFloat16``. + * **There is not even a spelling that would parse, and the parser says + so at the word.** Every one of these is ``SYNTAX_ERROR`` code 62, + ``failed at position N (UNSIGNED)``: + ``Decimal(10, 2) UNSIGNED``, ``Decimal(10, 0) UNSIGNED``, + ``Float32 UNSIGNED``, ``Float32(10) UNSIGNED``, ``Float64 UNSIGNED``. + The expected-token list the server prints back — ``COLLATE, NOT, + NULL, DEFAULT, MATERIALIZED, ALIAS, EPHEMERAL, AUTO_INCREMENT, + COMMENT, CODEC, STATISTICS, TTL, PRIMARY KEY, SETTINGS, token, + Comma, ClosingRoundBracket`` — is the whole of what may follow a + column type, and no attribute for signedness is on it. The + *inline* ``CHECK (c >= 0)`` is rejected by the **same** rule at the + **same** position, which is the clearest possible statement that the + slot does not exist: not "no ``UNSIGNED`` here" but "no attribute + here at all". + * **Signedness in the name means the integer columns are a different + kind of thing, not a decorated one.** ``CAST(-1 AS UInt8)`` is + ``255``; ``CAST(-1 AS Float32)``, ``CAST(-1 AS Float64)`` and + ``CAST(-1 AS Decimal(10,2))`` are all ``-1``. So the flag has nothing + to change on these three: no reachable value differs, and the only + way to make one differ is to leave the float family entirely. + + **Why the route forward is a named table-level constraint, not the + ``CHECK`` the other backends suggest** — that is also measured, and it + is a different answer: ``CREATE TABLE t (c Float32 CHECK (c >= 0))`` is + rejected with the same code-62 syntax error as ``UNSIGNED``, while + ``CREATE TABLE t (c Float32, CONSTRAINT cc CHECK (c >= 0))`` is + accepted and ``INSERT INTO t VALUES (-1)`` then fails with code 469, + ``Constraint 'cc' ... is violated ... Column values: c = -1``. So + ClickHouse does enforce the constraint — it just has nowhere to put an + inline one, which is the same fact as the missing attribute slot seen + from the other side. The ``suggestion`` below therefore names the + table-level form, because the inline form other backends recommend is + a syntax error here and telling a caller to write it would be worse + than saying nothing. + + Writing a bare ``Float32`` / ``Float64`` / ``Decimal(P, S)`` for an + unsigned request would create a column that accepts the negatives the + caller declared it would not, and report success: the same silent loss + as accepting the flag and discarding it. + + ``UnsupportedFeatureError``, not ``ValueError``: this is a declaration + the grammar cannot express at all rather than a wrong value, and the two + exceptions do not share a base class. This method runs **before** + ``_check_spelling`` in every caller, so a request wrong in two ways is + told about the signedness first — that one is a stronger and less + recoverable statement than a misspelled word. + """ + if not data_type.unsigned: + return + raise UnsupportedFeatureError( + self.name, + f"unsigned=True on {ch_word} " + f"(ClickHouse expresses signedness in the type NAME, not as a " + f"modifier on it, and only for integers: UInt8..UInt64 are their " + f"own type families, while {ch_word} has no unsigned variant at all " + f"-- system.data_type_families on 26.7 lists no unsigned float or " + f"decimal family, and '{ch_word} ... UNSIGNED' is a SYNTAX_ERROR at " + f"the word UNSIGNED, so there is no attribute slot to write it in)", + suggestion=( + f"Declare the {ch_word} column signed. ClickHouse has no " + f"unsigned {ch_word}: enforce the range with a named " + f"table-level constraint instead -- 'CONSTRAINT cc CHECK (c >= " + f"0)' in the column list, which the server enforces (code 469 " + f"VIOLATED_CONSTRAINT). An inline 'CHECK (c >= 0)' after the " + f"type is a syntax error on this server, as 'UNSIGNED' is." + ), + ) + def format_data_type_clickhouse_int8(self, data_type: ClickHouseInt8Type) -> Tuple[str, tuple]: + """ClickHouse ``Int8`` — the signed 8-bit integer. + + ``unsigned=True`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`, which is also where the reason is + written down. + """ + self._refuse_unsigned_integer(data_type, "Int8", "ClickHouseUInt8Type") return "Int8", () def format_data_type_clickhouse_int16(self, data_type: ClickHouseInt16Type) -> Tuple[str, tuple]: + """ClickHouse ``Int16`` — the signed 16-bit integer. + + ``unsigned=True`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`. + """ + self._refuse_unsigned_integer(data_type, "Int16", "ClickHouseUInt16Type") return "Int16", () def format_data_type_clickhouse_int32(self, data_type: ClickHouseInt32Type) -> Tuple[str, tuple]: + """ClickHouse ``Int32`` — the signed 32-bit integer. + + ``unsigned=True`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`. + """ + self._refuse_unsigned_integer(data_type, "Int32", "ClickHouseUInt32Type") return "Int32", () def format_data_type_clickhouse_int64(self, data_type: ClickHouseInt64Type) -> Tuple[str, tuple]: + """ClickHouse ``Int64`` — the signed 64-bit integer. + + ``unsigned=True`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`. + """ + self._refuse_unsigned_integer(data_type, "Int64", "ClickHouseUInt64Type") return "Int64", () def format_data_type_clickhouse_uint8(self, data_type: ClickHouseUInt8Type) -> Tuple[str, tuple]: + """ClickHouse ``UInt8`` — the unsigned 8-bit integer, [0 : 255]. + + The unsigned form is a *class* on this backend, so there is nothing to + refuse here: :class:`ClickHouseUInt8Type` pins ``unsigned=True`` in its own + ``__init__`` and takes no flag, which means this formatter cannot be reached + with ``unsigned=False``. + """ return "UInt8", () def format_data_type_clickhouse_uint16(self, data_type: ClickHouseUInt16Type) -> Tuple[str, tuple]: + """ClickHouse ``UInt16`` — the unsigned 16-bit integer. + + See :meth:`format_data_type_clickhouse_uint8` for why ``unsigned`` cannot + arrive here. + """ return "UInt16", () def format_data_type_clickhouse_uint32(self, data_type: ClickHouseUInt32Type) -> Tuple[str, tuple]: + """ClickHouse ``UInt32`` — the unsigned 32-bit integer. + + See :meth:`format_data_type_clickhouse_uint8` for why ``unsigned`` cannot + arrive here. + """ return "UInt32", () def format_data_type_clickhouse_uint64(self, data_type: ClickHouseUInt64Type) -> Tuple[str, tuple]: + """ClickHouse ``UInt64`` — the unsigned 64-bit integer. + + See :meth:`format_data_type_clickhouse_uint8` for why ``unsigned`` cannot + arrive here. + """ return "UInt64", () def format_data_type_clickhouse_float32(self, data_type: ClickHouseFloat32Type) -> Tuple[str, tuple]: @@ -150,10 +378,10 @@ def format_data_type_clickhouse_date(self, data_type: ClickHouseDateType) -> Tup def format_data_type_clickhouse_date32(self, data_type: ClickHouseDate32Type) -> Tuple[str, tuple]: return "Date32", () - def format_data_type_clickhouse_datetime(self, data_type: ClickHouseDateTimeType) -> Tuple[str, tuple]: + def format_data_type_clickhouse_timestamp(self, data_type: ClickHouseDateTimeType) -> Tuple[str, tuple]: return "DateTime", () - def format_data_type_clickhouse_datetime64(self, data_type: ClickHouseDateTime64Type) -> Tuple[str, tuple]: + def format_data_type_clickhouse_timestamp64(self, data_type: ClickHouseDateTime64Type) -> Tuple[str, tuple]: return f"DateTime64({data_type.precision})", () def format_data_type_clickhouse_bool(self, data_type: ClickHouseBoolType) -> Tuple[str, tuple]: @@ -268,7 +496,35 @@ def format_data_type_clickhouse_geometrycollection( return "GEOMETRYCOLLECTION", () def format_data_type_clickhouse_vector(self, data_type: ClickHouseVectorType) -> Tuple[str, tuple]: - return f"VECTOR({data_type.dim})", () + """Render ClickHouse's fixed-dimension vector as ``QBit(Float32, dim)``. + + ``QBit`` is ClickHouse's vector type, not ``VECTOR(n)``: ``VECTOR(n)`` is + MySQL 9.0's spelling and ClickHouse rejects it with ``Unknown data type + family: VECTOR`` — confirmed against 26.7.3.19 and against + ``system.data_type_families``, which lists ``QBit`` and no ``VECTOR``. + + Version gate + ------------ + ``QBit`` landed in 25.10 behind ``allow_experimental_qbit_type``, was + promoted to Beta in 26.1 and to General Availability in 26.2, after which + the setting is no longer needed — on 26.7 it already defaults to ``1``. + The gate is therefore GA, not first appearance: on 25.10–26.1 the type + exists but a column declaration still needs an experimental opt-in that + DDL does not carry, so this dialect declines to write one rather than + emit a statement the server will reject. + """ + if self.version < self._QBIT_MIN_VERSION: + required = "{}.{}+".format(*self._QBIT_MIN_VERSION[:2]) + raise UnsupportedFeatureError( + self.name, + f"the QBit vector type (needs ClickHouse {required})", + suggestion=( + "This server is older than QBit's General Availability " + f"({required}); store the embedding as Array(Float32) and " + "search it with L2Distance." + ), + ) + return f"QBit(Float32, {data_type.dim})", () # ------------------------------------------------------------------ # DataTypeSupport — supports_data_type_* (1:1 with format_data_type_*) @@ -328,10 +584,10 @@ def supports_data_type_clickhouse_date(self) -> bool: def supports_data_type_clickhouse_date32(self) -> bool: return True - def supports_data_type_clickhouse_datetime(self) -> bool: + def supports_data_type_clickhouse_timestamp(self) -> bool: return True - def supports_data_type_clickhouse_datetime64(self) -> bool: + def supports_data_type_clickhouse_timestamp64(self) -> bool: return True def supports_data_type_clickhouse_bool(self) -> bool: @@ -462,66 +718,312 @@ def supports_data_type_blob(self) -> bool: def supports_data_type_array(self) -> bool: return True + def supports_data_type_uuid(self) -> bool: + """ClickHouse has a native ``UUID`` type. + + Checked against the 26.7 scenario server: ``CREATE TABLE t (u UUID)`` + is accepted and ``system.columns`` reports ``UUID``. So the concept is + rendered here rather than substituted — the alternative would be to send a + UUID column to ``String``, which is what ``ClickHouseUUIDType`` avoided + doing. + """ + return True + # ------------------------------------------------------------------ # DataTypeSupport — suggested_data_types() # ------------------------------------------------------------------ def suggested_data_types(self) -> Dict[str, type]: + """Core concepts ClickHouse genuinely cannot spell, and what it stores. + + ClickHouse's data types are enumerated exhaustively in its manual, so a + concept is only absent from this dialect for a real reason, not because + the backend has not got round to it. Every substitute below is a + ``DataType`` subclass whose own ``name`` this dialect renders, so the + advice in ``format_data_type``'s error message can be acted on. + + ``custom`` + An unrecognised type name is a name this framework has no class + for. Some are names ClickHouse itself does not know, and the server + rejects those with ``Unknown data type family``; others are aliases + the server accepts but this parser deliberately does not claim + (``VARCHAR(10)`` resolves to ``String``, ``TINYINT`` to ``Int8``). + Either way there is no pass-through — the name would either fail at + the server or store a type the caller may not have meant — which is + why the dialect has no ``format_data_type_custom``: there is no + "write whatever the caller said" on this backend. The honest answer + is the byte-string type ClickHouse does have. + + ``interval`` + ClickHouse has no ``INTERVAL`` type. An elapsed time is a number of + seconds — an ``Int64`` — or, for a typed interval, the difference of + two ``DateTime64`` values. ``ClickHouseDateTime64Type`` is named for + the second form, but ``ClickHouseInt64Type`` is the one that covers + both, since the difference of two epoch counts is an epoch count. + + ``timestamptz`` + ClickHouse has no timezone-*suffixed* type. A timezone-aware instant + is a ``DateTime64`` that carries the zone in its declaration + (``DateTime64(3, 'UTC')``), which is why the substitute is the + class that knows how to render that form rather than a plain + ``DateTime``. Note the class as it stands carries no timezone + parameter, so the rendered column will use the server default — + see the merge-evaluation report. + + ``timetz`` + ClickHouse has no time-only type at all, with or without a zone; the + closest storage is a ``DateTime``, which is also what this dialect + renders for core's ``time`` (see :meth:`format_data_type_time`). + + ``xml`` + ClickHouse's manual enumerates its types and XML is not among them. + A document would be stored as a byte string. ``TextType`` is named + rather than ``ClickHouseStringType`` because the two render the same + ``String`` and the core class is the one that says what the value + means. + + ``binary`` / ``varbinary`` + ClickHouse's byte-string types are ``String`` (variable) and + ``FixedString(N)`` (fixed). ``FixedString`` is genuinely a + fixed-length byte string, so ``binary`` has a real counterpart; + ``varbinary`` does not, and ``String`` is what it would become. + + ``enum`` + ClickHouse has two enums, and neither is core's ``EnumType``: they + take explicit integers, cannot be NULL and leave most operations + undefined. See :class:`ClickHouseEnum8Type` for the full list of + differences. A generic enum's labels and nothing else cannot be + rendered as either, so the substitute is the plain byte string. + + ``jsonb`` + ClickHouse has a native ``JSON`` type, but it arrived in 24.10 and + ``JSONB`` — PostgreSQL's binary representation — has no counterpart; + the dialect renders ``json`` as ``String``, so ``jsonb`` is the same + storage. Deliberately *not* ``ClickHouseJSONType``: suggesting a + native type here would promise a feature whose version floor this + dialect does not check. + """ return { "binary": ClickHouseStringType, "varbinary": ClickHouseStringType, + "custom": ClickHouseStringType, "enum": ClickHouseStringType, + "interval": ClickHouseInt64Type, "jsonb": ClickHouseStringType, + "timetz": ClickHouseDateTimeType, + "timestamptz": ClickHouseDateTime64Type, + "xml": TextType, } # --- Core types (pure names) rendered to real ClickHouse SQL --- def format_data_type_integer(self, data_type: IntegerType) -> Tuple[str, tuple]: + """``INTEGER`` and ``INT`` are SQL's two spellings of one 4-byte signed + integer; ClickHouse has exactly one such type and calls it ``Int32``. + + Both spellings are accepted and both normalise to ``Int32``: the caller + asked for the concept, and refusing the non-default one would turn a + ``VarCharType(spelling="character varying")``-style choice into an error + without saying anything the rendered column does not already say. + + ``unsigned`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`, which is also where the reason is + written down. It is refused here for exactly the reason it is refused on + :meth:`format_data_type_clickhouse_int32`: ``IntegerType(unsigned=True)`` + and ``ClickHouseUInt32Type()`` are the same ``UInt32`` column, and this + backend keeps one class per column. + """ + self._check_spelling(data_type, IntegerType) + self._refuse_unsigned_integer(data_type, "Int32", "ClickHouseUInt32Type") return "Int32", () def format_data_type_bigint(self, data_type: BigIntType) -> Tuple[str, tuple]: + """``BIGINT`` and ``INT8`` are the standard's and PostgreSQL's spellings of + the 8-byte signed integer. ClickHouse writes ``Int64`` and has no other + word for it; both spellings are accepted and render as written above. + + ``unsigned`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`.""" + self._check_spelling(data_type, BigIntType) + self._refuse_unsigned_integer(data_type, "Int64", "ClickHouseUInt64Type") return "Int64", () def format_data_type_smallint(self, data_type: SmallIntType) -> Tuple[str, tuple]: + """``SMALLINT`` / ``INT2`` — the 2-byte signed integer, which ClickHouse + spells ``Int16``. Both spellings accepted. + + ``unsigned`` is refused rather than ignored; see + :meth:`_refuse_unsigned_integer`.""" + self._check_spelling(data_type, SmallIntType) + self._refuse_unsigned_integer(data_type, "Int16", "ClickHouseUInt16Type") return "Int16", () def format_data_type_tinyint(self, data_type: TinyIntType) -> Tuple[str, tuple]: + """``TINYINT`` / ``INT1`` — the 1-byte signed integer, which ClickHouse + spells ``Int8``. + + Neither spelling is ClickHouse's own (unlike ``BOOL``/``BOOLEAN``, which + ClickHouse does write), but both name the same 1-byte signed integer + ``Int8`` holds, so both are accepted and render ``Int8``. + + Note that ClickHouse *also* reads ``TINYINT UNSIGNED`` and ``INT1 + UNSIGNED`` — as aliases for ``UInt8``, which is the alias table's purpose. + That does not make the flag honoured on this concept: the alias resolves + to ``UInt8``, the column :class:`ClickHouseUInt8Type` already owns, so + ``unsigned`` is refused here and points at that class instead. See + :meth:`_refuse_unsigned_integer`. + """ + self._check_spelling(data_type, TinyIntType) + self._refuse_unsigned_integer(data_type, "Int8", "ClickHouseUInt8Type") return "Int8", () def format_data_type_varchar(self, data_type: VarCharType) -> Tuple[str, tuple]: + """ClickHouse has no length-bounded character type: ``String`` is + variable-length and unpadded, so the declared length is dropped. Both + ``VARCHAR`` and ``CHARACTER VARYING`` are accepted for the same reason as + the integer spellings — the concept was asked for and this is its only + storage.""" + self._check_spelling(data_type, VarCharType) return "String", () def format_data_type_char(self, data_type: CharType) -> Tuple[str, tuple]: + """``CHAR`` / ``CHARACTER`` — a fixed-length character string. + + Rendered ``String``, which is a **widening**, and the caller should know + it: ClickHouse has no fixed-width character type, so a ``CHAR(10)`` column + becomes an unbounded one. ``ClickHouseFixedStringType`` is the fixed-length + type on this backend, but it is a fixed number of *bytes*, zero-padded — + a different concept, so it is not what a generic ``CharType`` silently + becomes. Both spellings are accepted for the reason given on + :meth:`format_data_type_varchar`. + """ + self._check_spelling(data_type, CharType) return "String", () def format_data_type_text(self, data_type: TextType) -> Tuple[str, tuple]: + """``TEXT`` / ``CLOB`` — unbounded text, which is ``String``. + + ``CLOB`` is not a ClickHouse word, but it is the concept's own second + spelling and ``String`` is the concept's only storage here, so it is + accepted rather than refused. + """ + self._check_spelling(data_type, TextType) return "String", () def format_data_type_boolean(self, data_type: BooleanType) -> Tuple[str, tuple]: + """``BOOLEAN`` / ``BOOL`` — and both are ClickHouse's own words. + + Verified against the 26.7 scenario server: ``CREATE TABLE t (b BOOLEAN)`` + and ``(b Bool)`` are both accepted and both reported back as ``Bool``, so + this is the one concept whose core spelling list happens to be exactly + ClickHouse's vocabulary. + """ + self._check_spelling(data_type, BooleanType) return "Bool", () def format_data_type_date(self, data_type: DateType) -> Tuple[str, tuple]: + """``DATE`` — ClickHouse's own ``Date``, 1970..2149 in two bytes. + + Deliberately *not* ``Date32``: the wider range is a different storage and + a different type to the server, and silently choosing it would change the + range a column accepts. ``ClickHouseDate32Type`` is available for that. + """ return "Date", () def format_data_type_datetime(self, data_type: DateTimeType) -> Tuple[str, tuple]: + """``DATETIME`` — MySQL's wall-clock date and time. + + ClickHouse has no such type; its ``DateTime`` is a count of seconds since + the epoch. The nearest storage is ``DateTime``, and the difference is + visible: a ``DATETIME`` value has no epoch to convert against, so on this + backend it is stored as an instant. ``format_data_type_timestamp`` renders + the same word for core's ``TIMESTAMP``, which is the same storage and the + honest reading of it. + """ return "DateTime", () def format_data_type_timestamp(self, data_type: TimestampType) -> Tuple[str, tuple]: + """``TIMESTAMP`` — an instant, which is what ClickHouse's ``DateTime`` is. + + Sub-second precision is dropped: ClickHouse spells that + ``DateTime64(p)``, and choosing the word for the caller would silently + widen the value. ``ClickHouseDateTime64Type`` is the type that carries the + precision. + """ return "DateTime", () def format_data_type_time(self, data_type: TimeType) -> Tuple[str, tuple]: + """``TIME`` — time of day, with no date attached. + + ClickHouse has no time-only type; the closest storage is ``DateTime``, + which is what is rendered. A column declared ``TIME`` on this backend + therefore holds an instant, not a time of day. ``timetz`` is not rendered + at all — see :meth:`suggested_data_types`. + """ return "DateTime", () def format_data_type_float(self, data_type: FloatType) -> Tuple[str, tuple]: + """``FLOAT[(p)]`` — approximate numeric. + + Rendered ``Float32``, the default approximate type. A declared + ``precision`` is **not** honoured: ClickHouse's ``Float32`` has a fixed + 32-bit mantissa and there is no ``FLOAT(p)`` spelling to widen it to, so + honouring the request would mean substituting a different type for the one + asked for. ``format_data_type_double`` is where a caller who wants the + wider float should end up. + + ``unsigned`` is refused rather than ignored; see + :meth:`_refuse_unsigned_numeric`, which is where the reason, the + ``system.data_type_families`` inventory and the parser measurements are + written down. + """ + self._refuse_unsigned_numeric(data_type, "Float32") return "Float32", () def format_data_type_double(self, data_type: DoubleType) -> Tuple[str, tuple]: + """``DOUBLE`` / ``DOUBLE PRECISION`` — the 8-byte float, ``Float64``. + + Both spellings accepted; the concept's default spelling is ``double``, so + it must render, and the long form names the same 53-bit mantissa. + + ``unsigned`` is refused rather than ignored, before the spelling check; + see :meth:`_refuse_unsigned_numeric`. + """ + self._refuse_unsigned_numeric(data_type, "Float64") + self._check_spelling(data_type, DoubleType) return "Float64", () def format_data_type_real(self, data_type: RealType) -> Tuple[str, tuple]: + """``REAL`` — single precision. ClickHouse's word for it is ``Float32``, + which is also what ``FLOAT`` renders, because ClickHouse has no separate + ``REAL`` family. + + ``unsigned`` is refused rather than ignored; see + :meth:`_refuse_unsigned_numeric`. + """ + self._refuse_unsigned_numeric(data_type, "Float32") return "Float32", () def format_data_type_decimal(self, data_type: DecimalType) -> Tuple[str, tuple]: + """``DECIMAL`` / ``NUMERIC`` / ``DEC`` — exact fixed-point. + + ClickHouse's spelling vocabulary for this concept is *exactly* the core + list: verified against the 26.7 scenario server, ``NUMERIC(10,2)`` and + ``DEC(10,2)`` are both accepted and both reported back as + ``Decimal(10, 2)``. So all three are accepted and render the same. + + A precision of 38 is ClickHouse's maximum and is reached by + ``Decimal128``; ``Decimal(P, S)`` with a smaller P is a different storage, + which is why the precision is rendered rather than normalised. + + ``unsigned`` is refused rather than ignored, before the spelling check; + see :meth:`_refuse_unsigned_numeric`. ClickHouse's ``Decimal`` family + has no unsigned variant and no attribute slot — ``Decimal(10, 2) + UNSIGNED`` is a syntax error at the word — so the flag has nowhere to go + and the caller is told so by name rather than handed a signed column. + """ + self._refuse_unsigned_numeric(data_type, "Decimal(P, S)") + self._check_spelling(data_type, DecimalType) p = data_type.precision s = data_type.scale if p is not None and s is not None: @@ -531,12 +1033,47 @@ def format_data_type_decimal(self, data_type: DecimalType) -> Tuple[str, tuple]: return "Decimal(10, 0)", () def format_data_type_json(self, data_type: JsonType) -> Tuple[str, tuple]: + """``JSON`` — rendered ``String``, not ClickHouse's native ``JSON``. + + The native type exists (``ClickHouseJSONType`` renders it) but it arrived + in 24.10 and this dialect does not check the server version for it, so + the portable column is a byte string and a caller who wants the native + type asks for ``ClickHouseJSONType`` by name. Reporting this honestly + matters: a native ``JSON`` column knows its paths and ``JSONExtract`` + reads a field without parsing the value, and a ``String`` column does not. + """ return "String", () def format_data_type_blob(self, data_type: BlobType) -> Tuple[str, tuple]: + """``BLOB`` / ``BYTEA`` — a byte string, which is ClickHouse's ``String``. + + Both spellings accepted: ``BYTEA`` is PostgreSQL's word and ``BLOB`` is + the concept's default, and ``String`` is the concept's only unbounded byte + storage on this backend. + + Not ``FixedString``: there is no length to honour, and ``FixedString`` + zero-pads, so using it here would change the value. + """ + self._check_spelling(data_type, BlobType) return "String", () + def format_data_type_uuid(self, data_type: UUIDType) -> Tuple[str, tuple]: + """``UUID`` — ClickHouse has a type with that exact name and meaning. + + 128 bits, stored as two 64-bit halves. ``ClickHouseUUIDType`` renders the + same SQL through its own namespaced name; both are the UUID concept, one + reached through the core type and one through the backend's. + """ + return "UUID", () + def format_data_type_array(self, data_type: ArrayType) -> Tuple[str, tuple]: + """``T[]`` — ClickHouse's ``Array(T)``, always one dimension. + + ClickHouse has no multi-dimensional array syntax, so ``dimensions > 1`` + is rendered as a single level rather than rejected: a nested + ``Array(Array(T))`` is the same value, and this is the documented + ClickHouse spelling for it. + """ inner_sql, inner_params = self.format_data_type(data_type.element_type) return f"Array({inner_sql})", inner_params @@ -553,7 +1090,7 @@ def format_data_type_array(self, data_type: ArrayType) -> Tuple[str, tuple]: re.IGNORECASE, ) _CLICKHOUSE_DECIMAL_TYPES = re.compile( - r"^(?:Decimal(?:32|64|128)?(?:\(.*\))?)\b", + r"^(?:Decimal(?:32|64|128)?|Numeric|Dec)(?:\(.*\))?\b", re.IGNORECASE, ) _CLICKHOUSE_STRING_TYPES = re.compile( @@ -565,7 +1102,7 @@ def format_data_type_array(self, data_type: ArrayType) -> Tuple[str, tuple]: re.IGNORECASE, ) _CLICKHOUSE_OTHER_TYPES = re.compile( - r"^(?:Bool|UUID|IPv4|IPv6|JSON)\b", + r"^(?:Bool|Boolean|UUID|IPv4|IPv6|JSON)\b", re.IGNORECASE, ) _CLICKHOUSE_ENUM_TYPES = re.compile( @@ -586,14 +1123,230 @@ def format_data_type_array(self, data_type: ArrayType) -> Tuple[str, tuple]: re.IGNORECASE, ) _CLICKHOUSE_VECTOR_TYPES = re.compile( - r"^(?:VECTOR)\b", + r"^(?:QBit)\b", re.IGNORECASE, ) + #: MySQL-style integer aliases ClickHouse accepts and resolves to ``UInt``. + #: + #: One concept, one class: each alias maps to the ``ClickHouseUInt*Type`` + #: that already owns that column -- the same argument as + #: ``_refuse_unsigned_integer``'s "why the alias is not the answer" -- so an + #: aliased spelling reads back as the class the native spelling yields and a + #: differ comparing the two sees no change. + #: + #: Measured on the 26.7.3.19 scenario server. Every name below is + #: ``case_insensitive = 1`` in ``system.data_type_families`` with the + #: ``alias_to`` shown, and ``SELECT toTypeName(CAST(1 AS ))`` returns it: + #: + #: TINYINT UNSIGNED / INT1 UNSIGNED -> UInt8 + #: SMALLINT UNSIGNED / YEAR -> UInt16 + #: MEDIUMINT UNSIGNED / INT UNSIGNED / INTEGER UNSIGNED -> UInt32 + #: BIGINT UNSIGNED / UNSIGNED -> UInt64 + #: + #: ``YEAR`` is on the list because ClickHouse itself resolves it to + #: ``UInt16``; it is not a year storage type here. The MySQL-style ``YEAR`` + #: concept -- one byte, 1901..2155 -- lives in the MySQL backend as + #: ``MySQLYearType``, and mapping the word to a year class would claim a + #: storage the server does not use. + #: + #: Deliberately *not* on the list: the single-word signed aliases + #: (``TINYINT``, ``INT1``, ``SMALLINT``, ``MEDIUMINT``, ``INT``, ``INTEGER``, + #: ``BIGINT``) and their ``* SIGNED`` forms. The server accepts those too -- + #: measured on 26.7.3.19, ``toTypeName(CAST(1 AS TINYINT))`` is ``Int8`` -- + #: but they are spellings of the signed columns whose native names this + #: parser already claims, and the server's alias table is much wider than + #: this dialect's vocabulary (``VARCHAR(10)`` -> ``String`` and ``FLOAT`` -> + #: ``Float32`` are equally real and equally unclaimed). They keep the + #: answers they had: a single word falls through to ``CustomType``, a + #: two-word ``* SIGNED`` name keeps the closed-grammar ``InvalidTypeNameError``. + _CLICKHOUSE_UNSIGNED_INTEGER_ALIASES: Dict[str, type] = { + "TINYINT UNSIGNED": ClickHouseUInt8Type, + "INT1 UNSIGNED": ClickHouseUInt8Type, + "SMALLINT UNSIGNED": ClickHouseUInt16Type, + "YEAR": ClickHouseUInt16Type, + "MEDIUMINT UNSIGNED": ClickHouseUInt32Type, + "INT UNSIGNED": ClickHouseUInt32Type, + "INTEGER UNSIGNED": ClickHouseUInt32Type, + "BIGINT UNSIGNED": ClickHouseUInt64Type, + "UNSIGNED": ClickHouseUInt64Type, + } + + # ``UNSIGNED`` as a whole word, checked only against a string whose head one of + # the three numeric branches below would otherwise claim. The MySQL-style + # aliases above are recognised *before* this guard; none of their heads is a + # word those three branches claim, so the two checks cannot collide. What + # this catches is the opposite mistake: ``Int8 UNSIGNED``, ``Float32 + # UNSIGNED``, ``Float64 UNSIGNED`` and ``Decimal(10,2) UNSIGNED``, which a + # branch would otherwise match on the head and return a *signed* type for. + # See ``_refuse_unsigned_type_string``. + _UNSIGNED_ATTRIBUTE = re.compile(r"\bUNSIGNED\b", re.IGNORECASE) + + def _refuse_unsigned_type_string(self, raw: str) -> None: + """Refuse ``Int8`` / ``Float32`` / ``Float64`` / ``Decimal`` + ``UNSIGNED``. + + :meth:`_refuse_unsigned_numeric` closes the door on the way *in* — a + declaration. This closes it on the way *out*: a string. Without it the + three numeric branches below match on the word *in front of* the attribute + and return a signed type — ``parse_type("Float32 UNSIGNED")`` was + ``ClickHouseFloat32Type()`` and ``parse_type("Decimal(10,2) UNSIGNED")`` + was ``ClickHouseDecimalType(10, 2)`` — so a string saying ``UNSIGNED`` + produced a value object that ``==`` calls equal to the signed column. The + differ would then report no change for the one change the string + describes. + + It is checked *here*, at the head of ``parse_type``, and only for a string + whose head a numeric branch would claim. The MySQL-style integer aliases + ClickHouse genuinely accepts (``TINYINT UNSIGNED`` and friends, which the + server resolves to ``UInt8``…``UInt64``) are not attributes and are not + caught here: ``_CLICKHOUSE_UNSIGNED_INTEGER_ALIASES`` recognises them + before the numeric branches and returns the ``UInt`` class they name. + + No server can have written the strings this refuses: ``Int8 UNSIGNED`` is + ``UNKNOWN_TYPE`` code 50 on 26.7 and ``Float32 UNSIGNED`` / + ``Float64 UNSIGNED`` / ``Decimal(10,2) UNSIGNED`` are ``SYNTAX_ERROR`` + code 62 at the word — see :meth:`_refuse_unsigned_numeric`, which is where + the inventory and the parser output are recorded. So the branch guards + caller-supplied DDL, which ``parse_type`` is a documented reader of, and + says so rather than implying a catalog row is at stake. + """ + raise UnsupportedFeatureError( + self.name, + f"an UNSIGNED attribute on a ClickHouse type name ({raw!r}) " + f"(ClickHouse expresses signedness in the type NAME and only for " + f"integers: system.data_type_families on 26.7 resolves every " + f"'* UNSIGNED' name to a UInt family -- TINYINT/INT1 UNSIGNED -> " + f"UInt8, SMALLINT UNSIGNED -> UInt16, MEDIUMINT/INT/INTEGER UNSIGNED " + f"-> UInt32, BIGINT UNSIGNED -> UInt64 -- and has no unsigned float " + f"or decimal family at all, while Float32, Float64 and Decimal(P, S) " + f"carry no attribute in their own grammar. 'Int8 UNSIGNED' is " + f"UNKNOWN_TYPE (code 50) and 'Float32 UNSIGNED', 'Float64 UNSIGNED' " + f"and 'Decimal(10,2) UNSIGNED' are SYNTAX_ERROR (code 62) at the " + f"word, so the attribute is not merely absent: there is nowhere to " + f"put it.)", + suggestion=( + "Read the unsigned integer column back as the UInt family that " + "actually stores it -- 'UInt8', 'UInt16', 'UInt32', 'UInt64', or " + "the ClickHouseUInt*Type classes. For a float or a decimal there " + "is no unsigned column to read: declare it signed and enforce " + "the range with a named table-level 'CONSTRAINT cc CHECK (c >= " + "0)', which the server enforces (code 469 VIOLATED_CONSTRAINT)." + ), + ) def parse_type(self, raw: str) -> "DataType": + """Parse a ClickHouse type string back into a ``DataType``. + + One concept in, one class out: every recognised spelling of a concept + yields the *same* class, so no spelling becomes a synonym class and a + parsed type compares equal to the declared one that produced it. + + ClickHouse's own spelling vocabulary + ------------------------------------ + The spellings ClickHouse itself accepts for the concepts this dialect + models are recognised here, so that ``parse_type`` covers the server's + own output and the words its documentation lists: + + ``Bool`` / ``BOOLEAN`` + Both are accepted by the server and both reported back as ``Bool`` + (checked against 26.7). Same class either way. + + ``Decimal(P, S)`` / ``NUMERIC(P, S)`` / ``DEC(P, S)`` + All three accepted, all three reported back as ``Decimal(P, S)``. + That is exactly ``DecimalType.SPELLINGS``. + + ``TINYINT UNSIGNED`` / ``INT1 UNSIGNED`` / ``SMALLINT UNSIGNED`` / + ``MEDIUMINT UNSIGNED`` / ``INT UNSIGNED`` / ``INTEGER UNSIGNED`` / + ``BIGINT UNSIGNED`` / ``UNSIGNED`` / ``YEAR`` + The MySQL-style unsigned integer aliases. The server accepts every + one of them and resolves each to a ``UInt`` family (measured on + 26.7.3.19; the mapping is recorded in + ``_CLICKHOUSE_UNSIGNED_INTEGER_ALIASES``), so each yields the + ``ClickHouseUInt*Type`` that already owns that column — ``YEAR`` + included, which ClickHouse resolves to ``UInt16`` rather than to a + year type. ``TINYINT UNSIGNED`` is therefore a real spelling of an + unsigned column while ``Int8 UNSIGNED`` is not a spelling at all, + and the two get different answers. + + The rest of the server's alias table is deliberately not claimed. On + 26.7.3.19 ``system.data_type_families`` lists 73 aliases; ``FLOAT`` and + ``REAL`` resolve to ``Float32``, ``DOUBLE`` to ``Float64``, ``VARCHAR``, + ``TEXT`` and ``BLOB`` to ``String``, and the single-word integer aliases + (``TINYINT``, ``INT``, ``BIGINT``…) to the signed widths. The server + accepts every one of those words — ``CREATE TABLE t (a VARCHAR(10))`` + and ``(a TINYINT)`` are measured to succeed and store ``String`` and + ``Int8`` — but none is a word this parser claims: the vocabulary is the + native names this dialect renders. In the type grammar, case decides + only the native names, and only against their exact spelling: measured + on 26.7.3.19, ``(a Int8)`` and ``(a String)`` succeed while ``(a INT8)``, + ``(a int8)`` and ``(a string)`` are rejected with code 50 ``Unknown data + type family``; the alias words are ``case_insensitive = 1`` and accept + any case. + + That difference does not reach introspection. The catalog normalises + aliases at ``CREATE`` time — ``system.columns`` stores the resolved + type, not the spelling — so an aliased column comes back in its native + form (``TINYINT`` -> ``Int8``, ``VARCHAR(10)`` -> ``String``, + ``TINYINT UNSIGNED`` -> ``UInt8``, ``YEAR`` -> ``UInt16``) and the + introspection caller only ever hands this parser canonical names. An + unclaimed spelling therefore matters for hand-written DDL only, where it + falls through to :class:`~...types.custom.CustomType`, which this + dialect cannot render — a reported failure rather than a silently + written type the caller may not have meant. What a caller holding such + a name should use is answered by :meth:`suggested_data_types`, which is + also what the ``TypeError`` from an unrenderable ``CustomType`` points + at. + + Known lossiness + --------------- + ``DateTime(tz)`` and ``DateTime64(p, tz)`` carry a time zone that + :class:`ClickHouseDateTimeType` and + :class:`ClickHouseDateTime64Type` cannot hold, so the zone is dropped and + the parsed type renders without it. The server reports those forms, so an + introspected zone-aware column comes back as the server-default zone. + Carrying the zone is a change to those two classes' identity, and this + refactor does not change rendering — reported rather than fixed. + + ``QBit(element_type, dimension[, stride])`` loses two of its three + arguments for the same reason. The ``stride`` is dropped because it is a + storage detail that does not change what the vector *is*, and + ``element_type`` is dropped because + :class:`ClickHouseVectorType` models the dimension only; the formatter + renders ``Float32``. So a ``QBit(BFloat16, 1536)`` column introspects as + ``QBit(Float32, 1536)``. The dimension round-trips exactly, which is the + part that cannot change without rewriting the column. Making + ``element_type`` part of the class's identity is a wider change than + correcting a wrong spelling — reported rather than fixed. + + ``UNSIGNED`` as an attribute + Still refused rather than dropped, and this is the one word in the + grammar that *looks* like it might belong to a string: the integer, + float and decimal branches all match on the word in front of it, so + ``Float32 UNSIGNED`` used to parse to a signed + :class:`~...types.native.ClickHouseFloat32Type` — a value object + ``==`` calls equal to the signed column. The MySQL-style aliases + that carry the same word are the other case: ``TINYINT UNSIGNED`` + and friends name a column, so they are recognised as aliases above, + while bare ``UNSIGNED`` after a native name is an attribute nothing + can carry. See :meth:`_refuse_unsigned_type_string`. + """ stripped = raw.strip() upper = stripped.upper() + # The MySQL-style unsigned aliases: a closed list of words the server + # resolves to ``UInt*``. Checked first because none of them is a word + # the numeric branches below (or the UNSIGNED-attribute guard after it) + # claims, and a hit must not reach the ``CustomType`` fallback. + alias_class = self._CLICKHOUSE_UNSIGNED_INTEGER_ALIASES.get( + " ".join(upper.split())) + if alias_class is not None: + return alias_class(dialect=self) + + if (self._UNSIGNED_ATTRIBUTE.search(stripped) + and (self._CLICKHOUSE_INTEGER_TYPES.match(upper) + or self._CLICKHOUSE_FLOAT_TYPES.match(upper) + or self._CLICKHOUSE_DECIMAL_TYPES.match(upper))): + self._refuse_unsigned_type_string(stripped) + # Integer family if self._CLICKHOUSE_INTEGER_TYPES.match(upper): if upper.startswith("INT8"): @@ -622,22 +1375,27 @@ def parse_type(self, raw: str) -> "DataType": # Decimal family if self._CLICKHOUSE_DECIMAL_TYPES.match(upper): - nums = re.findall(r"\d+", stripped) + # The digits inside the parentheses are the arguments, never part of + # the type name: for ``Decimal32(S)`` the ``32`` is the width, so + # reading the first number out of the whole string would take the + # width for the scale and produce ``Decimal32(32)`` — or, for + # ``Decimal64(8)`` / ``Decimal128(18)``, a scale outside the range the + # constructor accepts, which is a hard failure rather than a wrong + # value. Take the arguments only. + inner = self._extract_inner_types(stripped) + nums = [int(n) for n in re.findall(r"\d+", " ".join(inner))] if upper.startswith("DECIMAL32"): - scale = int(nums[0]) if nums else 0 - return ClickHouseDecimal32Type(dialect=self, scale=scale) + return ClickHouseDecimal32Type(dialect=self, scale=nums[0] if nums else 0) if upper.startswith("DECIMAL64"): - scale = int(nums[0]) if nums else 0 - return ClickHouseDecimal64Type(dialect=self, scale=scale) + return ClickHouseDecimal64Type(dialect=self, scale=nums[0] if nums else 0) if upper.startswith("DECIMAL128"): - scale = int(nums[0]) if nums else 0 - return ClickHouseDecimal128Type(dialect=self, scale=scale) - # Decimal(P, S) - if len(nums) >= 2: - return ClickHouseDecimalType(dialect=self, precision=int(nums[0]), scale=int(nums[1])) - if len(nums) == 1: - return ClickHouseDecimalType(dialect=self, precision=int(nums[0])) - return ClickHouseDecimalType(dialect=self, precision=10, scale=0) + return ClickHouseDecimal128Type(dialect=self, scale=nums[0] if nums else 0) + # Decimal(P, S). ClickHouse writes the scale even when it is zero and + # reports ``Decimal(5)`` as ``Decimal(5, 0)``, so a one-argument + # declaration is parsed the way the server reports it back. + precision = nums[0] if nums else 10 + scale = nums[1] if len(nums) > 1 else 0 + return ClickHouseDecimalType(dialect=self, precision=precision, scale=scale) # String family if self._CLICKHOUSE_STRING_TYPES.match(upper): @@ -676,7 +1434,17 @@ def parse_type(self, raw: str) -> "DataType": # Enum types if self._CLICKHOUSE_ENUM_TYPES.match(upper): pairs = re.findall(r"'([^']*)'\s*=\s*(-?\d+)", stripped) - values = [(name, int(num)) for name, num in pairs] + # ClickHouse orders the labels of an enum by the integer assigned to + # each, not by the order they were declared in, and rewrites the + # declaration to match: a 26.7 server reports + # ``Enum8('x' = 1, 'y' = -2)`` back as ``Enum8('y' = -2, 'x' = 1)``. + # Sorting here is what makes an introspected enum equal to the + # declared one that produced it, instead of a schema diff reporting + # a change that is not there. + values = sorted( + ((name, int(num)) for name, num in pairs), + key=lambda pair: pair[1], + ) if upper.startswith("ENUM8"): return ClickHouseEnum8Type(dialect=self, values=values) return ClickHouseEnum16Type(dialect=self, values=values) @@ -693,7 +1461,35 @@ def parse_type(self, raw: str) -> "DataType": val_type = self.parse_type(inner[1]) return ClickHouseMapType(dialect=self, key_type=key_type, value_type=val_type) if upper.startswith("TUPLE") and inner: - types = [self.parse_type(t) for t in inner] + # Each element may carry a field name: ClickHouse writes + # ``Tuple(name Type, ...)``, and on 26.7.3.19 the names are + # part of the type's identity -- stored, reported, compared, + # used for ``tuple.name`` and INSERT matching (see the class + # docstring for the evidence). Feeding ``"name Type"`` to + # ``parse_type`` whole fell through to CustomType, whose + # type-name validation refused the space -- so the word this + # dialect renders was the one its own parse raised on. Split + # the leading identifier off when a second word follows; the + # names are carried into ``element_names`` so the round trip + # keeps them, which is why the class records them in + # PARAMETERS. + types = [] + names = [] + named = False + for part in inner: + words = part.split(None, 1) + if (len(words) == 2 + and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", words[0])): + names.append(words[0]) + types.append(self.parse_type(words[1])) + named = True + else: + names.append(None) + types.append(self.parse_type(part)) + if named: + return ClickHouseTupleType( + dialect=self, element_types=types, + element_names=names) return ClickHouseTupleType(dialect=self, element_types=types) if upper.startswith("NULLABLE") and inner: inner_type = self.parse_type(inner[0]) @@ -729,18 +1525,47 @@ def parse_type(self, raw: str) -> "DataType": "MULTIPOLYGON": ClickHouseMultiPolygonType, "GEOMETRYCOLLECTION": ClickHouseGeometryCollectionType, } - for name, cls in spatial_map.items(): - if upper.startswith(name): - return cls(dialect=self, srid=srid) + # Dispatch on the leading **word**, not on a prefix scanned in + # declaration order. ``GEOMETRYCOLLECTION`` begins with + # ``GEOMETRY``, and the map put ``GEOMETRY`` first, so the + # ``startswith`` loop read a collection column as the generic + # geometry type -- the same defect MariaDB's and MySQL's + # branches of this family each found and fixed, found here by + # the render-parse-re-render sweep. Every one of the eight words + # is a single identifier, so the leading word is the whole name. + head = re.match(r"[A-Z]+", upper) + word = head.group(0) if head else upper + return spatial_map.get(word, ClickHouseGeometryType)( + dialect=self, srid=srid) return ClickHouseGeometryType(dialect=self, srid=srid) - # Vector + # Vector: QBit(element_type, dimension[, stride]). + # + # The dimension is the first argument that is a *bare* integer. Scanning + # for any digit run instead would read the "32" out of "Float32" and + # report a QBit(Float32, 8) column as 32-dimensional — and the element + # type is not modelled by ClickHouseVectorType, so it is deliberately not + # matched here either. A strided QBit(Float32, 16, 8) still yields 16. if self._CLICKHOUSE_VECTOR_TYPES.match(upper): - nums = re.findall(r"\d+", stripped) - dim = int(nums[0]) if nums else 0 + inner = "" + if "(" in stripped and ")" in stripped: + inner = stripped[stripped.index("(") + 1: stripped.rindex(")")] + dim = next( + (int(arg.strip()) for arg in inner.split(",") if arg.strip().isdigit()), + 0, + ) return ClickHouseVectorType(dialect=self, dim=dim) - # Fallback + # Fallback: a type name this dialect does not know. It is a + # ``CustomType`` because that is honest — the framework has no class for + # it — but rendering one will raise, and that is the point rather than an + # oversight: an unclaimed name may be one ClickHouse rejects outright, or + # one it accepts as an alias (``VARCHAR(10)`` means ``String``), so + # writing ``data_type.raw`` back out would either fail at the server or + # silently store a type the caller may not have meant. There is no + # ``format_data_type_custom`` here, and + # ``suggested_data_types()["custom"]`` names ``ClickHouseStringType`` so + # the ``TypeError`` says what to use instead. from rhosocial.activerecord.backend.expression.types import CustomType return CustomType(dialect=self, raw=stripped) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/vector.py b/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/vector.py deleted file mode 100644 index b163fe8..0000000 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/mixins/vector.py +++ /dev/null @@ -1,74 +0,0 @@ -# src/rhosocial/activerecord/backend/impl/clickhouse/mixins/vector.py -from typing import Tuple - -from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError - - -class ClickHouseVectorMixin: - """ClickHouse does not support the MySQL 9.0 ``VECTOR`` type. - - ClickHouse has no ``VECTOR`` column type and no ``STRING_TO_VECTOR`` / - ``VECTOR_TO_STRING`` / ``VECTOR_DIM`` / ``DISTANCE_*`` function family. - Store embeddings as ``Array(Float32)`` and compute distance with - ClickHouse array functions (``L2Distance``, ``cosineDistance``). All - methods fail fast. - """ - - MAX_VECTOR_DIMENSION = 16384 - - def supports_vector_type(self) -> bool: - return False - - def supports_vector_index(self) -> bool: - return False - - def get_max_vector_dimension(self) -> int: - return self.MAX_VECTOR_DIMENSION - - def format_vector_literal(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR type", - suggestion="ClickHouse has no VECTOR type; use Array(Float32) + L2Distance.", - ) - - def format_string_to_vector(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "STRING_TO_VECTOR", - suggestion="ClickHouse has no VECTOR type; use Array(Float32).", - ) - - def format_vector_to_string(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR_TO_STRING", - suggestion="ClickHouse has no VECTOR type; use Array(Float32).", - ) - - def format_vector_dim(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR_DIM", - suggestion="ClickHouse has no VECTOR type; use length(Array).", - ) - - def format_distance_euclidean(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR distance functions", - suggestion="Use ClickHouse L2Distance on Array(Float32).", - ) - - def format_distance_cosine(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR distance functions", - suggestion="Use ClickHouse cosineDistance on Array(Float32).", - ) - - def format_distance_dot(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR distance functions", - suggestion="Use ClickHouse dotProduct on Array(Float32).", - ) - - def format_create_vector_index(self, expr) -> Tuple[str, tuple]: - raise UnsupportedFeatureError( - self.name, "VECTOR indexes", - suggestion="ClickHouse has no VECTOR indexes; use skip indexes (e.g. vector_similarity).", - ) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/__init__.py b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/__init__.py index 1f62ac2..18d5e68 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/__init__.py @@ -15,11 +15,11 @@ from .dml import ClickHouseDMLOperationSupport from .trigger import ClickHouseTriggerSupport from .table import ClickHouseTableSupport +from .type import ClickHouseTypeSupport from .partition import ClickHousePartitionSupport from .set_type import ClickHouseSetTypeSupport from .json import ClickHouseJSONFunctionSupport from .spatial import ClickHouseSpatialSupport -from .vector import ClickHouseVectorSupport from .fulltext_search import ClickHouseFullTextSearchSupport from .locking import ClickHouseLockingSupport from .modify_column import ClickHouseModifyColumnSupport @@ -36,11 +36,11 @@ "ClickHouseDMLOperationSupport", "ClickHouseTriggerSupport", "ClickHouseTableSupport", + "ClickHouseTypeSupport", "ClickHousePartitionSupport", "ClickHouseSetTypeSupport", "ClickHouseJSONFunctionSupport", "ClickHouseSpatialSupport", - "ClickHouseVectorSupport", "ClickHouseFullTextSearchSupport", "ClickHouseLockingSupport", "ClickHouseModifyColumnSupport", diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/partition.py b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/partition.py index 785b9c5..3667472 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/partition.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/partition.py @@ -5,289 +5,119 @@ concern per module so the protocol surface stays legible. """ -from typing import Any, Protocol, runtime_checkable, Tuple, TYPE_CHECKING +from typing import Protocol, runtime_checkable, Tuple, TYPE_CHECKING from rhosocial.activerecord.backend.dialect.protocols import PartitionSupport if TYPE_CHECKING: # pragma: no cover from rhosocial.activerecord.backend.impl.clickhouse.expression.partition import ( - ClickHouseAddPartitionExpression, - ClickHouseAnalyzePartitionExpression, - ClickHouseCheckPartitionExpression, - ClickHouseCoalescePartitionExpression, + ClickHouseAttachPartitionExpression, + ClickHouseDetachPartitionExpression, ClickHouseDropPartitionExpression, - ClickHouseExchangePartitionExpression, - ClickHouseGetPartitionsExpression, - ClickHouseOptimizePartitionExpression, - ClickHousePartitionByHash, - ClickHousePartitionByKey, - ClickHousePartitionByList, - ClickHousePartitionByListColumns, - ClickHousePartitionByRange, - ClickHousePartitionByRangeColumns, - ClickHousePartitionDefinition, - ClickHousePartitionMaxValue, - ClickHousePartitionValue, - ClickHouseRebuildPartitionExpression, - ClickHouseRemovePartitioningExpression, - ClickHouseReorganizePartitionExpression, - ClickHouseRepairPartitionExpression, - ClickHouseSubpartitionClause, - ClickHouseSubpartitionDefinition, - ClickHouseTruncatePartitionExpression, ) +__all__ = ["ClickHousePartitionSupport"] + + @runtime_checkable class ClickHousePartitionSupport(PartitionSupport, Protocol): """ClickHouse table partitioning protocol. - ClickHouse extends the generic PartitionSupport contract with ClickHouse-specific - partitioning strategies and ALTER TABLE partition maintenance statements. - Executable maintenance statements are represented by ClickHouse-specific - expressions and formatted by the methods declared here. + Feature Source: ClickHouse native — partitioning is a property of the + ``MergeTree`` family, not a ``PARTITION ... VALUES`` statement + + ClickHouse declares partitioning as an arbitrary expression in + ``CREATE TABLE``:: + + ENGINE = MergeTree PARTITION BY toYYYYMM(VisitDate) ORDER BY Hour + + so there is no strategy to select, no ``VALUES LESS THAN`` / ``VALUES IN`` / + ``MAXVALUE`` boundary, and no subpartitioning. That clause is written from + the table's ``storage_options`` mapping by + ``ClickHouseTableEngineMixin.format_table_engine_clauses``, not by a + partition formatter; the generic ``PartitionSupport`` members + (``format_partition_clause`` / ``format_partition_definition``) refuse + rather than invent a boundary syntax. + + Official Documentation: + - Custom Partitioning Key (the ``PARTITION BY`` clause, and ``system.parts``): + https://clickhouse.com/docs/engines/table-engines/mergetree-family/custom-partitioning-key + - ALTER TABLE ... PARTITION — the complete inventory of partition + operations, reproduced as ``ALTER_PARTITION_INVENTORY`` in + ``mixins/partition.py``: + https://clickhouse.com/docs/sql-reference/statements/alter/partition + + That inventory is also the evidence for the refusals. ClickHouse has no + ``ADD PARTITION`` (a partition materialises when a row lands in it), no + ``TRUNCATE PARTITION`` (the substitute is a light-weight mutation, + ``ALTER TABLE ... DELETE IN PARTITION ... WHERE ...``), and no + ``REORGANIZE`` / ``EXCHANGE`` / ``REMOVE PARTITIONING`` / ``COALESCE + PARTITION`` / ``ANALYZE|CHECK|OPTIMIZE|REBUILD|REPAIR PARTITION`` / + ``SUBPARTITION BY``. Checked directly on the 26.7.3.19 scenario server: + ``ALTER TABLE t TRUNCATE PARTITION p`` answers with an expectation list + enumerating every ``ALTER TABLE`` clause it accepts, and none of those + names is among them; ``ALTER TABLE t ADD PARTITION (...)`` answers + ``Expected one of: COLUMN, INDEX, STATISTICS, PROJECTION, CONSTRAINT``; + ``ALTER TABLE t SUBPARTITION BY HASH(id)`` gets the same list. + + Version Requirements: + - **None.** The three clauses below are MergeTree-family DDL and answer on + every ClickHouse release this backend targets; no version gate is + claimed, because none can be evidenced. """ - def supports_range_columns_partitioning(self) -> bool: - """Whether RANGE COLUMNS partitioning is supported.""" - ... - - def supports_list_columns_partitioning(self) -> bool: - """Whether LIST COLUMNS partitioning is supported.""" - ... - - def supports_key_table_partitioning(self) -> bool: - """Whether KEY partitioning is supported.""" - ... - - def supports_linear_hash_partitioning(self) -> bool: - """Whether LINEAR HASH partitioning is supported.""" - ... - - def supports_linear_key_partitioning(self) -> bool: - """Whether LINEAR KEY partitioning is supported.""" - ... - - def supports_partition_definition_options(self) -> bool: - """Whether partition definitions support extra ClickHouse options.""" - ... - - def supports_partition_value_maxvalue(self) -> bool: - """Whether MAXVALUE partition boundary token is supported.""" - ... - - def supports_add_partition(self) -> bool: - """Whether ADD PARTITION is supported.""" - ... - def supports_drop_partition(self) -> bool: - """Whether DROP PARTITION is supported.""" - ... - - def supports_truncate_partition(self) -> bool: - """Whether TRUNCATE PARTITION is supported.""" - ... - - def supports_reorganize_partition(self) -> bool: - """Whether REORGANIZE PARTITION is supported.""" - ... - - def supports_attach_partition(self) -> bool: - """Whether ATTACH PARTITION is supported.""" + """Whether ``ALTER TABLE ... DROP PARTITION ID`` is supported — it is.""" ... def supports_detach_partition(self) -> bool: - """Whether DETACH PARTITION is supported.""" + """Whether ``ALTER TABLE ... DETACH PARTITION ID`` is supported — it is.""" ... - def supports_remove_partitioning(self) -> bool: - """Whether ALTER TABLE ... REMOVE PARTITIONING is supported.""" - ... - - def supports_coalesce_partition(self) -> bool: - """Whether ALTER TABLE ... COALESCE PARTITION is supported.""" - ... - - def supports_exchange_partition(self) -> bool: - """Whether ALTER TABLE ... EXCHANGE PARTITION is supported.""" - ... - - def supports_analyze_partition(self) -> bool: - """Whether ALTER TABLE ... ANALYZE PARTITION is supported.""" - ... - - def supports_check_partition(self) -> bool: - """Whether ALTER TABLE ... CHECK PARTITION is supported.""" - ... - - def supports_optimize_partition(self) -> bool: - """Whether ALTER TABLE ... OPTIMIZE PARTITION is supported.""" - ... - - def supports_rebuild_partition(self) -> bool: - """Whether ALTER TABLE ... REBUILD PARTITION is supported.""" - ... - - def supports_repair_partition(self) -> bool: - """Whether ALTER TABLE ... REPAIR PARTITION is supported.""" - ... - - def format_partition_definition(self, definition: "ClickHousePartitionDefinition") -> Tuple[str, tuple]: - """Format a ClickHouse PARTITION definition.""" - ... - - def format_partition_definition_options(self, options: dict) -> Tuple[str, tuple]: - """Format ClickHouse PARTITION definition options.""" + def supports_attach_partition(self) -> bool: + """Whether ``ALTER TABLE ... ATTACH PARTITION ID`` is supported — it is.""" ... - def format_get_partitions_expression(self, expr: "ClickHouseGetPartitionsExpression") -> Tuple[str, tuple]: - """Format a ``SELECT ... FROM information_schema.PARTITIONS`` query. + def format_drop_partition_statement( + self, expr: "ClickHouseDropPartitionExpression" + ) -> Tuple[str, tuple]: + """Format ``ALTER TABLE ... DROP PARTITION ID ''``. Args: - expr: ClickHouseGetPartitionsExpression with the target table name. + expr: expression carrying the table and the ``system.parts`` + partition id. Returns: - Tuple of (SQL string, parameters tuple). + Tuple of (SQL string, empty params tuple). """ ... - def format_partition_value( - self, - expr: "ClickHousePartitionValue | ClickHousePartitionMaxValue", + def format_detach_partition_statement( + self, expr: "ClickHouseDetachPartitionExpression" ) -> Tuple[str, tuple]: - """Format a ClickHouse partition boundary value.""" - ... - - def format_subpartition_by(self, expr: "ClickHouseSubpartitionClause") -> Tuple[str, tuple]: - """Format ``SUBPARTITION BY {HASH|KEY}(...) SUBPARTITIONS N``. + """Format ``ALTER TABLE ... DETACH PARTITION ID ''``. Args: - expr: ClickHouseSubpartitionClause with strategy, optional expression, - optional count, and optional explicit definitions. + expr: expression carrying the table and the ``system.parts`` + partition id. Returns: - Tuple of (SQL string, parameters tuple). - - Raises: - UnsupportedFeatureError: if subpartitioning is not supported. + Tuple of (SQL string, empty params tuple). """ ... - def format_subpartition_definition(self, definition: "ClickHouseSubpartitionDefinition") -> Tuple[str, tuple]: - """Format a single ``SUBPARTITION name ...`` clause. + def format_attach_partition_statement( + self, expr: "ClickHouseAttachPartitionExpression" + ) -> Tuple[str, tuple]: + """Format ``ALTER TABLE ... ATTACH PARTITION ID ''``. Args: - definition: ClickHouseSubpartitionDefinition with name and typed - partition options. + expr: expression carrying the table and the ``system.parts`` + partition id. Returns: - Tuple of (SQL string, parameters tuple). - - Raises: - ValueError: if the definition name is empty. + Tuple of (SQL string, empty params tuple). """ ... - - def format_partition_by_range(self, expr: "ClickHousePartitionByRange") -> Tuple[str, tuple]: - """Format PARTITION BY RANGE.""" - ... - - def format_partition_by_range_columns(self, expr: "ClickHousePartitionByRangeColumns") -> Tuple[str, tuple]: - """Format PARTITION BY RANGE COLUMNS.""" - ... - - def format_partition_by_list(self, expr: "ClickHousePartitionByList") -> Tuple[str, tuple]: - """Format PARTITION BY LIST.""" - ... - - def format_partition_by_list_columns(self, expr: "ClickHousePartitionByListColumns") -> Tuple[str, tuple]: - """Format PARTITION BY LIST COLUMNS.""" - ... - - def format_partition_by_hash(self, expr: "ClickHousePartitionByHash") -> Tuple[str, tuple]: - """Format PARTITION BY HASH or LINEAR HASH.""" - ... - - def format_partition_by_key(self, expr: "ClickHousePartitionByKey") -> Tuple[str, tuple]: - """Format PARTITION BY KEY or LINEAR KEY.""" - ... - - def format_add_partition_statement(self, expr: "ClickHouseAddPartitionExpression") -> Tuple[str, tuple]: - """Format ALTER TABLE ... ADD PARTITION.""" - ... - - def format_drop_partition_statement(self, expr: "ClickHouseDropPartitionExpression") -> Tuple[str, tuple]: - """Format ALTER TABLE ... DROP PARTITION.""" - ... - - def format_truncate_partition_statement(self, expr: "ClickHouseTruncatePartitionExpression") -> Tuple[str, tuple]: - """Format ALTER TABLE ... TRUNCATE PARTITION.""" - ... - - def format_reorganize_partition_statement( - self, - expr: "ClickHouseReorganizePartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... REORGANIZE PARTITION.""" - ... - - def format_exchange_partition_statement( - self, - expr: "ClickHouseExchangePartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... EXCHANGE PARTITION.""" - ... - - def format_remove_partitioning_statement( - self, - expr: "ClickHouseRemovePartitioningExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... REMOVE PARTITIONING.""" - ... - - def format_coalesce_partition_statement( - self, - expr: "ClickHouseCoalescePartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... COALESCE PARTITION.""" - ... - - def format_analyze_partition_statement( - self, - expr: "ClickHouseAnalyzePartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... ANALYZE PARTITION.""" - ... - - def format_check_partition_statement( - self, - expr: "ClickHouseCheckPartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... CHECK PARTITION.""" - ... - - def format_optimize_partition_statement( - self, - expr: "ClickHouseOptimizePartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... OPTIMIZE PARTITION.""" - ... - - def format_rebuild_partition_statement( - self, - expr: "ClickHouseRebuildPartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... REBUILD PARTITION.""" - ... - - def format_repair_partition_statement( - self, - expr: "ClickHouseRepairPartitionExpression", - ) -> Tuple[str, tuple]: - """Format ALTER TABLE ... REPAIR PARTITION.""" - ... - - def format_partition_name_list(self, expr) -> Tuple[str, tuple]: - """Format a list of partition names: `p0`, `p1`, ...""" - ... diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/type.py b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/type.py new file mode 100644 index 0000000..afed4f3 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/type.py @@ -0,0 +1,48 @@ +# src/rhosocial/activerecord/backend/impl/clickhouse/protocols/type.py +"""ClickHouse-native data type protocol. + +Split out of the former single-module ``clickhouse/protocols.py``; one +concern per module so the protocol surface stays legible. +""" + +from typing import Protocol, runtime_checkable + +from rhosocial.activerecord.backend.dialect.protocols import DataTypeSupport + + +__all__ = ["ClickHouseTypeSupport"] + + +@runtime_checkable +class ClickHouseTypeSupport(DataTypeSupport, Protocol): + """ClickHouse-native data type protocol. + + Feature Source: ClickHouse native (not SQL standard) + + ClickHouse owns a large set of types core does not model, and the framework + renders them through the naming convention rather than through an interface: + a ``ClickHouse`` type declares ``name = "clickhouse_"`` and the dialect + answers with ``format_data_type_clickhouse_`` / + ``supports_data_type_clickhouse_``. Enumerating all of them here would + mean restating the type list in a second place, where it would go stale the + next time a type is added. + + What the protocol *does* state is the one thing the naming pattern cannot + express: that this backend's own types are declared alongside the core ones + in the same correspondence, and that a concept ClickHouse genuinely cannot + spell is answered through + :meth:`~...dialect.mixins.data_type.DataTypeMixin.suggested_data_types` + rather than left silent. ``ClickHouseSetTypeSupport`` is *not* part of this — + it covers ``FIND_IN_SET`` and the SET literal syntax, which are functions + rather than a column type, and mixing the two would make ``isinstance`` true + for a dialect that had one and not the other. + """ + + def suggested_data_types(self) -> dict: + """Core concepts this dialect cannot spell, and the types it stores instead. + + Keys are core concept ``name``s; values are ``DataType`` subclasses this + dialect renders through their own ``name``. Disjoint from the set of + rendered names by contract. + """ + ... diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/vector.py b/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/vector.py deleted file mode 100644 index c6222d1..0000000 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/protocols/vector.py +++ /dev/null @@ -1,73 +0,0 @@ -# src/rhosocial/activerecord/backend/impl/clickhouse/protocols/vector.py -"""ClickHouse vector data type protocol. - -Split out of the former single-module ``clickhouse/protocols.py``; one -concern per module so the protocol surface stays legible. -""" - -from typing import Protocol, runtime_checkable, Tuple, TYPE_CHECKING - -@runtime_checkable -class ClickHouseVectorSupport(Protocol): - """ClickHouse vector data type protocol. - - Feature Source: ClickHouse 8.0+ (optional feature in 8.0.16+, GA in 8.0.17+) - - ClickHouse vector features: - - VECTOR data type for embedding vectors - - Vector operations and functions - - Official Documentation: - - Vector Type: https://dev.clickhouse.com/doc/refman/8.0/en/vector-type.html - - Version Requirements: - - VECTOR type: ClickHouse 8.0.16+ (experimental), 8.0.17+ (GA) - """ - - def supports_vector_type(self) -> bool: - """Whether VECTOR data type is supported (ClickHouse 8.0.17+).""" - ... - - def supports_vector_index(self) -> bool: - """Whether vector index is supported (ClickHouse 8.0.17+).""" - ... - - def get_max_vector_dimension(self) -> int: - """Get the maximum supported vector dimension. - - Returns: - Maximum number of dimensions supported for VECTOR type - """ - ... - - def format_vector_literal(self, expr) -> Tuple[str, tuple]: - """Format vector literal from a list of float values.""" - ... - - def format_string_to_vector(self, expr) -> Tuple[str, tuple]: - """Format STRING_TO_VECTOR function call.""" - ... - - def format_vector_to_string(self, expr) -> Tuple[str, tuple]: - """Format VECTOR_TO_STRING function call.""" - ... - - def format_vector_dim(self, expr) -> Tuple[str, tuple]: - """Format VECTOR_DIM function call to get vector dimension.""" - ... - - def format_distance_euclidean(self, expr) -> Tuple[str, tuple]: - """Format EUCLIDEAN_DISTANCE function call.""" - ... - - def format_distance_cosine(self, expr) -> Tuple[str, tuple]: - """Format COSINE_DISTANCE function call.""" - ... - - def format_distance_dot(self, expr) -> Tuple[str, tuple]: - """Format DOT_PRODUCT function call.""" - ... - - def format_create_vector_index(self, expr) -> Tuple[str, tuple]: - """Format CREATE VECTOR INDEX statement.""" - ... diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/show/__init__.py b/src/rhosocial/activerecord/backend/impl/clickhouse/show/__init__.py index 4e5be0a..ff6f7a6 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/show/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/show/__init__.py @@ -6,15 +6,18 @@ - Expression classes for SHOW commands - Dialect mixin for SQL generation - Functionality classes for executing SHOW commands -- Backend mixins for show() method +- Backend mixins for the ``show()`` factory method (see the caveat below) Usage: - # Via backend - result = backend.show().create_table("users") - columns = backend.show().columns("users", full=True) - indexes = backend.show().indexes("users") - tables = backend.show().tables() - databases = backend.show().databases() + # The reachable path is the show sub-introspector, which is sync or async + # according to the backend it was created for: + create = backend.introspector.show.create_table("users") + tables = backend.introspector.show.tables() + databases = backend.introspector.show.databases() + create = await async_backend.introspector.show.create_table("users") + + # backend.show() is NOT wired up on this backend -- see show/backend_mixin.py. + # The classes below are still exported for API compatibility. """ from ..expression.show import ( diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/show/backend_mixin.py b/src/rhosocial/activerecord/backend/impl/clickhouse/show/backend_mixin.py index 6593e2c..db3f3a2 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/show/backend_mixin.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/show/backend_mixin.py @@ -1,16 +1,21 @@ # src/rhosocial/activerecord/backend/impl/clickhouse/show/backend_mixin.py """ -ClickHouse backend mixins for SHOW functionality. - -This module provides mixin classes that add the show() factory method -to ClickHouse backends. The show() method returns a ClickHouseShowFunctionality -instance that provides all ClickHouse SHOW commands. +ClickHouse backend mixins for the ``show()`` factory method. .. warning:: - This module was copied from the MySQL backend template and contains - MySQL-style SQL functions/show commands. ClickHouse uses different - function names (e.g. ``JSONExtract*``) and a different SHOW command - subset. May generate non-ClickHouse SQL; verify before use. + **Neither mixin is mixed into a ClickHouse backend.** Neither + ``ClickHouseBackend`` nor ``AsyncClickHouseBackend`` lists them in its MRO, + so ``backend.show()`` raises ``AttributeError``; verified by introspection, + not by reading. The reachable path into this package is + ``backend.introspector.show`` (``SyncShowIntrospector`` / + ``AsyncShowIntrospector`` in ``..introspection.show_introspector``), which + builds the same expressions through the same dialect mixin and carries its + own parsers. + + They are kept because they are importable, documented API: deleting them + would be a breaking change for any caller importing them directly, whereas + mixing them in would change ``backend.show()`` from an ``AttributeError`` + into a working call and so deserves its own decision. """ from typing import TYPE_CHECKING @@ -23,7 +28,8 @@ class ClickHouseShowMixin: """ClickHouse backend mixin for SHOW functionality. Provides the show() factory method that returns a ClickHouseShowFunctionality - instance for executing ClickHouse SHOW commands. + instance for executing ClickHouse SHOW commands. Not currently composed + into a backend — see the module warning. """ def show(self) -> "ClickHouseShowFunctionality": @@ -34,7 +40,8 @@ def _create_show_functionality(self) -> "ClickHouseShowFunctionality": """Create ClickHouse SHOW functionality instance. Returns: - ClickHouseShowFunctionality instance with version awareness. + ClickHouseShowFunctionality instance. The version it records comes + from the backend and is not compared against any threshold. """ from .functionality import ClickHouseShowFunctionality @@ -51,18 +58,22 @@ def _create_show_functionality(self) -> "ClickHouseShowFunctionality": class AsyncClickHouseShowMixin: """Async ClickHouse backend mixin for SHOW functionality. - ClickHouse backend is synchronous-only (clickhouse-connect is a sync-only - driver), so no async SHOW functionality is provided. + Refuses rather than providing an async implementation, because the + ``clickhouse-connect`` driver is synchronous. Not currently composed into + ``AsyncClickHouseBackend`` either; the async backend reaches this package + through ``AsyncShowIntrospector``. """ def show(self): """Raise NotImplementedError: async SHOW functionality is not supported.""" raise NotImplementedError( - "ClickHouse backend is synchronous-only; async SHOW functionality is not supported." + "ClickHouse show functionality is synchronous; use " + "await backend.introspector.show.() instead." ) def _create_show_functionality(self): """Raise NotImplementedError: async SHOW functionality is not supported.""" raise NotImplementedError( - "ClickHouse backend is synchronous-only; async SHOW functionality is not supported." + "ClickHouse show functionality is synchronous; use " + "await backend.introspector.show.() instead." ) diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/show/dialect.py b/src/rhosocial/activerecord/backend/impl/clickhouse/show/dialect.py index 011d340..cd0790a 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/show/dialect.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/show/dialect.py @@ -12,14 +12,34 @@ - Generate SQL string and parameter tuple - Return (sql, params) tuple -.. warning:: - This module was copied from the MySQL backend template and contains - MySQL-style SQL functions/show commands. ClickHouse uses different - function names (e.g. ``JSONExtract*``) and a different SHOW command - subset. May generate non-ClickHouse SQL; verify before use. +Nine of these are ClickHouse statements and produce SQL: +``SHOW CREATE TABLE``, ``SHOW CREATE VIEW``, ``SHOW [FULL] TABLES``, +``SHOW DATABASES``, ``SHOW [FULL] COLUMNS``, ``SHOW INDEX``, +``SHOW PROCESSLIST``, ``SHOW ENGINES`` and ``SHOW GRANTS`` (all nine verified +against the 26.7.3.19 scenario server; the ``SHOW`` reference page at +https://clickhouse.com/docs/reference/statements/show documents each). +The rest of the MySQL ``SHOW`` set is ClickHouse's to not have; each raises +``UnsupportedFeatureError`` naming the ClickHouse replacement, so no method +here emits SQL this server would reject. + +Two places where ClickHouse's own grammar is narrower than MySQL's, and the +rendering has to respect that rather than MySQL's shape: + +* ``SHOW COLUMNS`` / ``SHOW INDEX`` take the database as part of the table + reference (``SHOW COLUMNS FROM db.t``), never as a second ``FROM``. The + documentation lists ``[{FROM | IN} ]``, but 26.7.3.19 answers + ``SHOW INDEX FROM t FROM db`` with + ``Code: 62 ... Expected one of: ParserArrayOfJSONIdentifierDelimiter, token + sequence, OpeningSquareBracket, Dot, token, WHERE, INTO OUTFILE, FORMAT, + SETTINGS, ParallelWithClause, PARALLEL WITH, end of query``; the abbreviated + ``db.t`` form is the one the server accepts. +* ``SHOW PROCESSLIST`` has no ``FULL`` variant (26.7.3.19: + ``Code: 62 ... Expected one of: DATABASES, CLUSTERS, MERGES, FILESYSTEM + CACHES, CLUSTER, CHANGED, SETTINGS, TEMPORARY, TABLES, DICTIONARIES, COLUMNS, + FIELDS``), so ``full=True`` cannot be rendered and is not rendered. """ -from typing import Tuple, TYPE_CHECKING +from typing import Optional, Tuple, TYPE_CHECKING from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError from rhosocial.activerecord.backend.expression.objects import Table, View @@ -57,6 +77,24 @@ class ClickHouseShowDialectMixin: This mixin is added to ClickHouseDialect to provide SHOW functionality. """ + # ========== Shared helpers ========== + + def _qualified_table(self, table: str, schema: Optional[str] = None) -> str: + """Render ``table`` or ``db.table`` as one identifier reference. + + ``SHOW COLUMNS`` and ``SHOW INDEX`` in ClickHouse take the database + inside the table reference (``SHOW COLUMNS FROM db.t``); the two-``FROM`` + form MySQL allows is a ``SYNTAX_ERROR`` on 26.7.3.19. Measured error: + ``Code: 62. DB::Exception: Syntax error: failed at position 34 (FROM): + FROM test_db. Expected one of: ParserArrayOfJSONIdentifierDelimiter, + token sequence, OpeningSquareBracket, Dot, token, WHERE, INTO OUTFILE, + FORMAT, SETTINGS, ParallelWithClause, PARALLEL WITH, end of query. + (SYNTAX_ERROR)``. + """ + if schema: + return f"{self.format_identifier(schema)}.{self.format_identifier(table)}" + return self.format_identifier(table) + # ========== SHOW CREATE Statements ========== def format_show_create_table(self, expr: "ShowCreateTableExpression") -> Tuple[str, tuple]: @@ -95,28 +133,50 @@ def format_show_create_trigger(self, expr: "ShowCreateTriggerExpression") -> Tup # ========== SHOW COLUMNS/INDEX ========== def format_show_columns(self, expr: "ShowColumnsExpression") -> Tuple[str, tuple]: - """Format SHOW [FULL] COLUMNS statement. + """Format SHOW [FULL] COLUMNS FROM
[LIKE ]. - Note: - MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement and answers it with lower-cased column + names — on 26.7.3.19 ``SHOW COLUMNS FROM db.t`` reports ``field``, + ``type``, ``null``, ``key``, ``default``, ``extra``, and adds + ``collation``, ``comment`` and ``privileges`` under ``FULL``. Bare + ``SHOW COLUMNS`` (no table) is a ``SYNTAX_ERROR``: the server expects + ``FROM`` or ``IN`` after the keyword. + + ``EXTENDED`` is also accepted and documented but has no effect, so it is + not rendered. """ - raise UnsupportedFeatureError( - self.name, - "SHOW COLUMNS", - suggestion="Use DESCRIBE TABLE or query system.columns instead.", - ) + params = expr.get_params() + table = params["table"] + schema = params.get("schema") + full = params.get("full", False) + like_pattern = params.get("like_pattern") + + parts = ["SHOW"] + if full: + parts.append("FULL") + parts.append("COLUMNS FROM") + parts.append(self._qualified_table(table, schema)) + + sql_params = () + if like_pattern: + parts.append(f"LIKE {self.p()}") + sql_params = (like_pattern,) + + return " ".join(parts), sql_params def format_show_index(self, expr: "ShowIndexExpression") -> Tuple[str, tuple]: - """Format SHOW INDEX statement. + """Format SHOW INDEX FROM
. - Note: - MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement, explicitly "for compatibility with MySQL", + and answers it with ClickHouse's own column names — notably ``pk_col`` + where MySQL says ``Column_name``, and ``index_type`` carrying + ``PRIMARY`` / ``MINMAX`` / ``BLOOM_FILTER`` / ``SET`` / ... rather than + ``BTREE``. See ``https://clickhouse.com/docs/reference/statements/show``. """ - raise UnsupportedFeatureError( - self.name, - "SHOW INDEX", - suggestion="Query system.data_skipping_indices or system.tables instead.", - ) + params = expr.get_params() + table = params["table"] + schema = params.get("schema") + return f"SHOW INDEX FROM {self._qualified_table(table, schema)}", () # ========== SHOW TABLES/DATABASES ========== @@ -205,16 +265,19 @@ def format_show_status(self, expr: "ShowStatusExpression") -> Tuple[str, tuple]: # ========== SHOW PROCESSLIST/WARNINGS/ERRORS ========== def format_show_processlist(self, expr: "ShowProcessListExpression") -> Tuple[str, tuple]: - """Format SHOW PROCESSLIST statement. - - Note: - MySQL-only command, not supported by ClickHouse. + """Format SHOW PROCESSLIST. + + ClickHouse's ``SHOW PROCESSLIST`` is ``SELECT * FROM system.processes``: + it answers with that table's 43 lower-cased columns (``query_id``, + ``user``, ``address``, ``port``, ``elapsed``, ``current_database``, + ``query``, ...). ``full`` is accepted and ignored because ClickHouse has + no ``FULL`` variant — on 26.7.3.19 ``SHOW FULL PROCESSLIST`` is + ``Code: 62 ... Expected one of: DATABASES, CLUSTERS, MERGES, FILESYSTEM + CACHES, CLUSTER, CHANGED, SETTINGS, TEMPORARY, TABLES, DICTIONARIES, + COLUMNS, FIELDS. (SYNTAX_ERROR)``, and the statement already reports + the full query text in ``query``. """ - raise UnsupportedFeatureError( - self.name, - "SHOW PROCESSLIST", - suggestion="Query system.processes instead.", - ) + return "SHOW PROCESSLIST", () def format_show_warnings(self, expr: "ShowWarningsExpression") -> Tuple[str, tuple]: """Format SHOW WARNINGS statement. @@ -243,16 +306,19 @@ def format_show_errors(self, expr: "ShowErrorsExpression") -> Tuple[str, tuple]: # ========== SHOW ENGINES/CHARSET/COLLATION ========== def format_show_engines(self, expr: "ShowEnginesExpression") -> Tuple[str, tuple]: - """Format SHOW ENGINES statement. - - Note: - MySQL-only command, not supported by ClickHouse. + """Format SHOW ENGINES. + + ClickHouse's ``SHOW ENGINES`` is ``SELECT * FROM system.table_engines``. + Its columns are its own, not MySQL's: ``name`` plus eight + ``supports_*`` flags (``supports_settings``, + ``supports_skipping_indices``, ``supports_projections``, + ``supports_sort_order``, ``supports_ttl``, ``supports_replication``, + ``supports_deduplication``, ``supports_parallel_insert``) and then + ``description``, ``syntax``, ``examples``, ``introduced_in`` and + ``related``. MySQL's ``Engine`` / ``Support`` / ``Transactions`` / ``XA`` + / ``Savepoints`` appear nowhere in that list. """ - raise UnsupportedFeatureError( - self.name, - "SHOW ENGINES", - suggestion="Query system.table_engines instead.", - ) + return "SHOW ENGINES", () def format_show_charset(self, expr: "ShowCharsetExpression") -> Tuple[str, tuple]: """Format SHOW CHARACTER SET statement. @@ -281,16 +347,23 @@ def format_show_collation(self, expr: "ShowCollationExpression") -> Tuple[str, t # ========== SHOW GRANTS/PLUGINS ========== def format_show_grants(self, expr: "ShowGrantsExpression") -> Tuple[str, tuple]: - """Format SHOW GRANTS statement. - - Note: - MySQL-only command, not supported by ClickHouse. + """Format SHOW GRANTS [FOR ]. + + ClickHouse's grammar is ``SHOW GRANTS [FOR user1 [, user2 ...]] [WITH + IMPLICIT] [FINAL]`` — a comma-separated list of access-entity names, + with **no** ``user@host`` form: ``SHOW GRANTS FOR root@localhost`` on + 26.7.3.19 is ``Code: 511. DB::Exception: There is no role + `root@localhost` in `user directories`. (UNKNOWN_ROLE)``, because + ClickHouse identifies access entities by role name alone. A supplied + ``host`` is therefore not rendered; ``FOR `` is the whole + statement. ``SHOW GRANTS FOR CURRENT USER`` is also a ``SYNTAX_ERROR`` + here; omitting ``FOR`` is how the current user's grants are read. """ - raise UnsupportedFeatureError( - self.name, - "SHOW GRANTS", - suggestion="Query system.grants or other system.* access control tables instead.", - ) + params = expr.get_params() + user = params.get("user") + if not user: + return "SHOW GRANTS", () + return f"SHOW GRANTS FOR {self.format_identifier(user)}", () def format_show_plugins(self, expr: "ShowPluginsExpression") -> Tuple[str, tuple]: """Format SHOW PLUGINS statement. diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/show/functionality.py b/src/rhosocial/activerecord/backend/impl/clickhouse/show/functionality.py index e27d653..ae933bf 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/show/functionality.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/show/functionality.py @@ -12,11 +12,38 @@ - Executes SQL via backend.execute() - Parses results into typed dataclasses -.. warning:: - This module was copied from the MySQL backend template and contains - MySQL-style SQL functions/show commands. ClickHouse uses different - function names (e.g. ``JSONExtract*``) and a different SHOW command - subset. May generate non-ClickHouse SQL; verify before use. +What this module renders, and what it refuses +--------------------------------------------- +Nine of these commands are ClickHouse statements and are rendered, each read +back from the column names the 26.7.3.19 server actually sends: + +====================== ======================================== +``SHOW CREATE TABLE`` ``ClickHouseShowDialectMixin`` +``SHOW CREATE VIEW`` (single ``statement`` output column — + verified on 26.7.3.19) +``SHOW [FULL] TABLES`` ``name`` / ``engine`` output columns +``SHOW DATABASES`` ``name`` output column +``SHOW [FULL] COLUMNS`` ``field`` / ``type`` / ``null`` / ``key`` / + ``default`` / ``extra`` (+ ``collation``, + ``comment``, ``privileges`` under FULL) +``SHOW INDEX`` ``table`` / ``key_name`` / ``pk_col`` / + ``index_type`` / ``expression`` / ... +``SHOW PROCESSLIST`` ``system.processes``' 43 columns +``SHOW ENGINES`` ``name`` + eight ``supports_*`` flags + + ``description`` / ``syntax`` / ``introduced_in`` +``SHOW GRANTS`` one column named after the statement text +====================== ======================================== + +Every other command in the MySQL ``SHOW`` set is ClickHouse's to not have, +and its ``format_show_*`` raises ``UnsupportedFeatureError`` naming the +ClickHouse replacement — ``SHOW TABLE STATUS`` → ``system.tables``, +``SHOW VARIABLES`` → ``system.settings``, ``SHOW WARNINGS`` → +``system.query_log``, and so on. Their result dataclasses stay declared so the +API is complete, but nothing can reach them. + +``version`` is taken from the connection the caller already opened (the +backend reports it through ``get_server_version()``); it is not compared +against a capability threshold anywhere in this file. """ from typing import Optional, Tuple, TYPE_CHECKING @@ -24,10 +51,15 @@ from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError from ..expression.show import ( + ShowColumnsExpression, ShowCreateTableExpression, ShowCreateViewExpression, - ShowTablesExpression, ShowDatabasesExpression, + ShowEnginesExpression, + ShowGrantsExpression, + ShowIndexExpression, + ShowProcessListExpression, + ShowTablesExpression, ) if TYPE_CHECKING: @@ -37,8 +69,16 @@ class ClickHouseShowFunctionality: """ClickHouse-specific SHOW functionality implementation. - Provides all ClickHouse SHOW commands using expression-dialect pattern. - Supports version-aware feature detection for ClickHouse 5.7 vs 8.0 differences. + Provides the nine SHOW commands ClickHouse has, using the + expression-dialect pattern; every other one refuses. + + No capability here is gated on a version. The ``version`` argument is kept + for interface symmetry with the other backends' show functionality and for + logging, not for feature detection: this module previously held a + "ClickHouse 5.7 vs 8.0" distinction and an ``_supports_invisible_columns`` + flag tested as ``version >= (8, 0, 0)``. ClickHouse's numbering has never + had a 5.7 or an 8.0, no SHOW command was branched on either, and the flag + was read by nothing in this repository. """ def __init__(self, backend: "ClickHouseBackend", version: Optional[Tuple[int, ...]] = None): @@ -46,111 +86,62 @@ def __init__(self, backend: "ClickHouseBackend", version: Optional[Tuple[int, .. Args: backend: ClickHouseBackend instance for executing queries. - version: ClickHouse server version tuple, e.g., (8, 0, 0) for ClickHouse 8.0. + version: ClickHouse server version tuple as the backend reports it, + e.g. ``(26, 7, 3)`` for 26.7.3.19. Recorded, not compared. """ self._backend = backend self._version = version self.dialect = backend.dialect - # ClickHouse 8.0+ supports invisible columns - self._supports_invisible_columns = version >= (8, 0, 0) if version else True # ========== Parsing Helper Methods ========== + # + # Every one of these delegates to the corresponding parser on + # :class:`...introspection.show_introspector.ShowMixin`, so there is exactly + # one implementation of "which ClickHouse column goes in which field" in this + # package — reachable from both the sync/async show sub-introspector and this + # ``ShowFunctionality`` surface, without the two drifting apart. - def _parse_create_table_result(self, result, table: str): - """Parse SHOW CREATE TABLE result.""" - from .types import ShowCreateTableResult + @staticmethod + def _show_parser(): + from ..introspection.show_introspector import ShowMixin - if not result.data or len(result.data) == 0: - return None + return ShowMixin - row = result.data[0] - return ShowCreateTableResult( - table_name=row.get("Table", row.get("TABLE", table)), - create_statement=row.get("Create Table", row.get("CREATE TABLE", "")), - ) + def _parse_create_table_result(self, result, table: str): + """Parse ``SHOW CREATE TABLE`` — a single ``statement`` column.""" + return self._show_parser()._parse_create_table(result.data, table) def _parse_create_view_result(self, result, view_name: str): - """Parse SHOW CREATE VIEW result.""" - from .types import ShowCreateViewResult - - if not result.data or len(result.data) == 0: - return None - - row = result.data[0] - return ShowCreateViewResult( - view_name=row.get("View", row.get("VIEW", view_name)), - create_statement=row.get("Create View", row.get("CREATE VIEW", "")), - character_set_client=row.get("character_set_client"), - collation_connection=row.get("collation_connection"), - ) + """Parse ``SHOW CREATE VIEW`` — a single ``statement`` column, as above.""" + return self._show_parser()._parse_create_view(result.data, view_name) def _parse_columns_result(self, result): - """Parse SHOW COLUMNS result.""" - from .types import ShowColumnResult - - columns = [] - for row in result.data: - col = ShowColumnResult( - field=row.get("Field", row.get("COLUMN_NAME")), - type=row.get("Type", row.get("COLUMN_TYPE")), - null=row.get("Null", row.get("IS_NULLABLE")), - key=row.get("Key", row.get("COLUMN_KEY")), - default=row.get("Default", row.get("COLUMN_DEFAULT")), - extra=row.get("Extra", row.get("EXTRA")), - ) - # FULL mode additional fields - if "Collation" in row or "Privileges" in row: - col.privileges = row.get("Privileges") - col.comment = row.get("Comment") - columns.append(col) - return columns + """Parse ``SHOW [FULL] COLUMNS`` — ``field`` / ``type`` / ``null`` / + ``key`` / ``default`` / ``extra``, plus the three ``FULL`` columns.""" + return self._show_parser()._parse_columns(result.data) def _parse_indexes_result(self, result): - """Parse SHOW INDEX result.""" - from .types import ShowIndexResult - - indexes = [] - for row in result.data: - indexes.append( - ShowIndexResult( - table_name=row.get("Table", row.get("TABLE_NAME")), - non_unique=row.get("Non_unique", row.get("NON_UNIQUE")), - key_name=row.get("Key_name", row.get("INDEX_NAME")), - seq_in_index=row.get("Seq_in_index", row.get("SEQ_IN_INDEX")), - column_name=row.get("Column_name", row.get("COLUMN_NAME")), - collation=row.get("Collation", row.get("COLLATION")), - cardinality=row.get("Cardinality", row.get("CARDINALITY")), - sub_part=row.get("Sub_part", row.get("SUB_PART")), - packed=row.get("Packed", row.get("PACKED")), - null=row.get("Null", row.get("NULLABLE")), - index_type=row.get("Index_type") or row.get("INDEX_TYPE") or "BTREE", - comment=row.get("Comment", row.get("INDEX_COMMENT")), - index_comment=row.get("Index_comment", row.get("INDEX_COMMENT")), - visible=row.get("Visible", row.get("IS_VISIBLE")), - expression=row.get("Expression", row.get("EXPRESSION")), - ) - ) - return indexes + """Parse ``SHOW INDEX`` — ``table`` / ``key_name`` / ``pk_col`` / ...""" + return self._show_parser()._parse_indexes(result.data) def _parse_tables_result(self, result): - """Parse SHOW TABLES result.""" + """Parse ``SHOW [FULL] TABLES`` — columns ``name`` and, under ``FULL``, ``engine``. + + ``ShowTableResult.table_type`` therefore carries the ClickHouse storage + engine name, not MySQL's ``BASE TABLE`` / ``VIEW``. + """ from .types import ShowTableResult - tables = [] - for row in result.data: - if len(row) == 1: - tables.append(ShowTableResult(name=list(row.values())[0], table_type=None)) - else: - name_key = next((k for k in row.keys() if k.startswith("Tables_in_")), None) - if name_key: - tables.append(ShowTableResult(name=row[name_key], table_type=row.get("Table_type"))) - return tables + return [ + ShowTableResult(name=row.get("name"), table_type=row.get("engine")) + for row in result.data + ] def _parse_databases_result(self, result): - """Parse SHOW DATABASES result.""" + """Parse ``SHOW DATABASES`` — the column is ``name``, not ``Database``.""" from .types import ShowDatabaseResult - return [ShowDatabaseResult(name=row.get("Database")) for row in result.data] + return [ShowDatabaseResult(name=row.get("name")) for row in result.data] def _parse_table_status_result(self, result): """Parse SHOW TABLE STATUS result.""" @@ -246,24 +237,8 @@ def _parse_status_result(self, result): ] def _parse_processlist_result(self, result): - """Parse SHOW PROCESSLIST result.""" - from .types import ShowProcessListResult - - processes = [] - for row in result.data: - processes.append( - ShowProcessListResult( - id=row.get("Id", row.get("ID")), - user=row.get("User"), - host=row.get("Host"), - command=row.get("Command"), - time=row.get("Time"), - db=row.get("db"), - state=row.get("State"), - info=row.get("Info"), - ) - ) - return processes + """Parse ``SHOW PROCESSLIST`` — ``system.processes``' own columns.""" + return self._show_parser()._parse_processlist(result.data) def _parse_warnings_result(self, result): """Parse SHOW WARNINGS result.""" @@ -292,21 +267,8 @@ def _parse_errors_result(self, result): ] def _parse_engines_result(self, result): - """Parse SHOW ENGINES result.""" - from .types import ShowEngineResult - - engines = [] - for row in result.data: - engines.append( - ShowEngineResult( - engine=row.get("Engine"), - support=row.get("Support"), - transactions=row.get("Transactions"), - xa=row.get("XA"), - savepoints=row.get("Savepoints"), - ) - ) - return engines + """Parse ``SHOW ENGINES`` — ``name`` plus the eight ``supports_*`` flags.""" + return self._show_parser()._parse_engines(result.data) def _parse_charset_result(self, result): """Parse SHOW CHARACTER SET result.""" @@ -339,10 +301,8 @@ def _parse_collation_result(self, result): ] def _parse_grants_result(self, result): - """Parse SHOW GRANTS result.""" - from .types import ShowGrantResult - - return [ShowGrantResult(grants=row.get("Grants for")) for row in result.data] + """Parse ``SHOW GRANTS`` — one column, named after the statement text.""" + return self._show_parser()._parse_grants(result.data) def _parse_plugins_result(self, result): """Parse SHOW PLUGINS result.""" @@ -411,28 +371,36 @@ def columns( ): """Get column information for a table. - Note: - MySQL-only command, not supported by ClickHouse. + Renders ``SHOW [FULL] COLUMNS FROM [.]
[LIKE ]`` + and reads ClickHouse's own output columns. See + ``https://clickhouse.com/docs/reference/statements/show``. """ - raise UnsupportedFeatureError( - self._backend.dialect.name, - "SHOW COLUMNS", - suggestion="Use DESCRIBE TABLE or query system.columns instead.", - ) + expr = ShowColumnsExpression(self.dialect, table) + if schema: + expr.schema(schema) + if full: + expr.full() + if like: + expr.like(like) + sql, params = expr.to_sql() + result = self._backend.execute(sql, params) + return self._parse_columns_result(result) # ========== SHOW INDEX ========== def indexes(self, table: str, schema: Optional[str] = None): """Get index information for a table. - Note: - MySQL-only command, not supported by ClickHouse. + Renders ``SHOW INDEX FROM [.]
`` and reads ClickHouse's own + output columns — the primary key columns plus one row per data skipping + index. See ``https://clickhouse.com/docs/reference/statements/show``. """ - raise UnsupportedFeatureError( - self._backend.dialect.name, - "SHOW INDEX", - suggestion="Query system.data_skipping_indices or system.tables instead.", - ) + expr = ShowIndexExpression(self.dialect, table) + if schema: + expr.schema(schema) + sql, params = expr.to_sql() + result = self._backend.execute(sql, params) + return self._parse_indexes_result(result) # ========== SHOW TABLES ========== @@ -556,14 +524,17 @@ def status(self, like: Optional[str] = None, session: bool = True): def processlist(self, full: bool = False): """Show process list. - Note: - MySQL-only command, not supported by ClickHouse. + Renders ``SHOW PROCESSLIST``, which in ClickHouse is + ``SELECT * FROM system.processes``. ``full`` is accepted for interface + symmetry and ignored: the statement has no ``FULL`` variant and already + reports the full query text in ``query``. """ - raise UnsupportedFeatureError( - self._backend.dialect.name, - "SHOW PROCESSLIST", - suggestion="Query system.processes instead.", - ) + expr = ShowProcessListExpression(self.dialect) + if full: + expr.full() + sql, params = expr.to_sql() + result = self._backend.execute(sql, params) + return self._parse_processlist_result(result) # ========== SHOW WARNINGS/ERRORS ========== @@ -596,14 +567,15 @@ def errors(self, limit: Optional[int] = None): def engines(self): """Show storage engines. - Note: - MySQL-only command, not supported by ClickHouse. + Renders ``SHOW ENGINES``, which in ClickHouse is + ``SELECT * FROM system.table_engines``; the result carries the engine name, + the eight ``supports_*`` capability flags and the engine's own + ``description`` / ``syntax`` / ``introduced_in`` documentation columns. """ - raise UnsupportedFeatureError( - self._backend.dialect.name, - "SHOW ENGINES", - suggestion="Query system.table_engines instead.", - ) + expr = ShowEnginesExpression(self.dialect) + sql, params = expr.to_sql() + result = self._backend.execute(sql, params) + return self._parse_engines_result(result) # ========== SHOW CHARSET ========== @@ -638,14 +610,22 @@ def collation(self, like: Optional[str] = None): def grants(self, user: Optional[str] = None, host: Optional[str] = None): """Show grants. - Note: - MySQL-only command, not supported by ClickHouse. + Renders ``SHOW GRANTS`` or ``SHOW GRANTS FOR ``. ClickHouse + identifies access entities by role name alone and has no ``user@host`` + form — ``SHOW GRANTS FOR root@localhost`` is + ``Code: 511. DB::Exception: There is no role `root@localhost` in `user + directories`. (UNKNOWN_ROLE)`` on 26.7.3.19 — so a supplied ``host`` is + not rendered. + + The statement returns a single column whose name follows the output + format, so the parser reads the column's value rather than a named key. """ - raise UnsupportedFeatureError( - self._backend.dialect.name, - "SHOW GRANTS", - suggestion="Query system.grants or other system.* access control tables instead.", - ) + expr = ShowGrantsExpression(self.dialect) + if user: + expr.for_user(user, host) + sql, params = expr.to_sql() + result = self._backend.execute(sql, params) + return self._parse_grants_result(result) # ========== SHOW PLUGINS ========== diff --git a/src/rhosocial/activerecord/backend/impl/clickhouse/show/types.py b/src/rhosocial/activerecord/backend/impl/clickhouse/show/types.py index cbae7a1..d146555 100644 --- a/src/rhosocial/activerecord/backend/impl/clickhouse/show/types.py +++ b/src/rhosocial/activerecord/backend/impl/clickhouse/show/types.py @@ -5,15 +5,54 @@ This module defines result dataclasses for ClickHouse SHOW commands. These types provide structured access to SHOW command output. -.. warning:: - This module was copied from the MySQL backend template and contains - MySQL-style SQL functions/show commands. ClickHouse uses different - function names (e.g. ``JSONExtract*``) and a different SHOW command - subset. May generate non-ClickHouse SQL; verify before use. +Which of them can actually be filled +------------------------------------ +Every statement modelled here is one this backend emits and parses against +26.7.3.19, and each result dataclass reads the column names the server actually +sent: + +================================= ========================================= +statement ClickHouse output columns +================================= ========================================= +``SHOW CREATE TABLE`` / ``VIEW`` ``statement`` +``SHOW [FULL] COLUMNS`` ``field``, ``type``, ``null``, ``key``, + ``default``, ``extra`` (+ ``collation``, + ``comment``, ``privileges`` under FULL) +``SHOW INDEX`` ``table``, ``non_unique``, ``key_name``, + ``seq_in_index``, ``pk_col``, ``collation``, + ``cardinality``, ``sub_part``, ``packed``, + ``null``, ``index_type``, ``comment``, + ``index_comment``, ``visible``, + ``expression`` +``SHOW [FULL] TABLES`` ``name`` (+ ``engine`` under FULL) +``SHOW DATABASES`` ``name`` +``SHOW PROCESSLIST`` ``system.processes``'s 43 columns +``SHOW ENGINES`` ``name``, eight ``supports_*`` flags, + ``description``, ``syntax``, ``examples``, + ``introduced_in``, ``related`` +``SHOW GRANTS`` one column, named after the statement text + plus the requested output format +================================= ========================================= + +Where the two databases disagree it is not cosmetic: reading MySQL's names +against ClickHouse's rows does not raise, it silently yields ``None``. Two of +those disagreements are structural rather than cosmetic and are documented on +the dataclasses — ``SHOW INDEX`` calls the indexed column ``pk_col``, and +``SHOW COLUMNS`` packs ``ALIAS`` / ``DEFAULT`` / ``MATERIALIZED`` expressions +all into ``default``. + +The remaining dataclasses mirror MySQL's ``SHOW`` result shapes for statements +ClickHouse genuinely does not have. Their dialect formatters raise +``UnsupportedFeatureError`` naming a ClickHouse replacement, which is the +negotiation point a caller needs when switching backends. + +There is deliberately no dataclass for a statement this package has no path to +at all — no expression class, no dialect formatter, no parser and no export: +those were removed rather than left as types nothing can construct. """ from dataclasses import dataclass -from typing import Optional +from typing import Any, Optional # ==================== CREATE Statement Results ==================== @@ -43,8 +82,11 @@ class ShowCreateViewResult: Attributes: view_name: Name of the view. create_statement: Complete CREATE VIEW statement. - character_set_client: Client character set. - collation_connection: Connection collation. + character_set_client: Always ``None`` on this backend. ClickHouse + answers ``SHOW CREATE VIEW`` with a single ``statement`` column; + the MySQL ``character_set_client`` / ``collation_connection`` + columns are not produced. + collation_connection: Always ``None`` on this backend — see above. """ view_name: str @@ -57,7 +99,11 @@ class ShowCreateViewResult: class ShowCreateTriggerResult: """Result from SHOW CREATE TRIGGER command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse. ``SHOW CREATE TRIGGER`` + is a ``SYNTAX_ERROR`` on 26.7.3.19 and the server has no triggers at all; + ``format_show_create_trigger`` refuses with "ClickHouse does not support + triggers". A SQL UDF (``CREATE FUNCTION ... AS``) invoked from the + application is the ClickHouse equivalent. Contains the CREATE TRIGGER statement for a trigger. @@ -76,52 +122,6 @@ class ShowCreateTriggerResult: database_collation: Optional[str] = None -@dataclass -class ShowCreateProcedureResult: - """Result from SHOW CREATE PROCEDURE command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Contains the CREATE PROCEDURE statement. - - Attributes: - procedure_name: Name of the procedure. - create_statement: Complete CREATE PROCEDURE statement. - character_set_client: Client character set. - collation_connection: Connection collation. - database_collation: Database collation. - """ - - procedure_name: str - create_statement: str - character_set_client: Optional[str] = None - collation_connection: Optional[str] = None - database_collation: Optional[str] = None - - -@dataclass -class ShowCreateFunctionResult: - """Result from SHOW CREATE FUNCTION command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Contains the CREATE FUNCTION statement. - - Attributes: - function_name: Name of the function. - create_statement: Complete CREATE FUNCTION statement. - character_set_client: Client character set. - collation_connection: Connection collation. - database_collation: Database collation. - """ - - function_name: str - create_statement: str - character_set_client: Optional[str] = None - collation_connection: Optional[str] = None - database_collation: Optional[str] = None - - # ==================== Column Information Results ==================== @@ -129,19 +129,29 @@ class ShowCreateFunctionResult: class ShowColumnResult: """Result from SHOW [FULL] COLUMNS command. - NOTE: MySQL-only command, not supported by ClickHouse. ClickHouse uses DESCRIBE TABLE. - - Contains column information for a table. + ClickHouse has this statement. On 26.7.3.19 ``SHOW COLUMNS FROM db.t`` + answers with the lower-cased column names ``field``, ``type``, ``null``, + ``key``, ``default``, ``extra``, and adds ``collation``, ``comment`` and + ``privileges`` under ``FULL`` — the same list the ``SHOW`` reference page + documents, so the mapping below is one-to-one. Attributes: - field: Column name. + field: Column name (ClickHouse's spelling; MySQL's is ``Field``). type: Column data type. - null: Whether the column allows NULL ('YES' or 'NO'). - key: Key type ('PRI', 'UNI', 'MUL', or empty). - default: Default value for the column. - extra: Additional information (auto_increment, etc.). - privileges: Column privileges (FULL mode only). - comment: Column comment (FULL mode only). + null: ``'YES'`` if the type is ``Nullable``, ``'NO'`` otherwise. + key: ``'PRI SOR'`` for a column in the sorting key, ``'PRI'`` for a + primary-key-only column, ``''`` otherwise. Not MySQL's + ``PRI``/``UNI``/``MUL`` — ClickHouse has no unique-constraint + concept to report here. + default: The expression of an ``ALIAS``, ``DEFAULT`` **or** + ``MATERIALIZED`` column — the server reports all three through this + one column — and ``None`` for a plain column. + extra: Always ``''``: the server documents this column as unused. + collation: Always ``None``: ClickHouse has no per-column collations. + ``FULL`` only. + comment: Column comment. ``FULL`` only. + privileges: Column privilege string; the server documents this as + "currently not available", and it is ``''``. ``FULL`` only. """ field: str @@ -150,8 +160,9 @@ class ShowColumnResult: key: str default: Optional[str] = None extra: Optional[str] = None - privileges: Optional[str] = None + collation: Optional[str] = None comment: Optional[str] = None + privileges: Optional[str] = None # ==================== Table Status Results ==================== @@ -161,7 +172,10 @@ class ShowColumnResult: class ShowTableStatusResult: """Result from SHOW TABLE STATUS command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse: ``SHOW TABLE STATUS`` + is the one MySQL ``SHOW`` the server answers with ``UNKNOWN_TABLE``-style + failure rather than a bare parse error on 26.7.3.19. ``format_show_table_status`` + refuses and points at ``system.tables``. Contains extensive table metadata. @@ -213,27 +227,41 @@ class ShowTableStatusResult: class ShowIndexResult: """Result from SHOW INDEX command. - NOTE: MySQL-only command, not supported by ClickHouse. - - Contains index information for a table. - Each row represents one column in an index. + ClickHouse has this statement — the ``SHOW`` reference page says it "mostly + exists for compatibility with MySQL" — and answers with ClickHouse's own + column names: ``table``, ``non_unique``, ``key_name``, ``seq_in_index``, + ``pk_col``, ``collation``, ``cardinality``, ``sub_part``, ``packed``, + ``null``, ``index_type``, ``comment``, ``index_comment``, ``visible``, + ``expression``. Note ``pk_col``, which MySQL spells ``Column_name`` (the + reference page still calls it ``column_name``; the 26.7.3.19 server sends + ``pk_col``, and the server is what the parser reads). + + One statement, two kinds of row: the table's **primary key columns** + (``key_name='PRIMARY'``, ``pk_col`` holding the column name, + ``collation='A'``, ``seq_in_index`` counting from 1) and one row per **data + skipping index** (``key_name`` = the index name, ``pk_col=''``, + ``seq_in_index`` always 1, the indexed expression in ``expression``). Attributes: - table_name: Table name. - non_unique: 1 if index allows duplicates, 0 if unique. - key_name: Index name. - seq_in_index: Column sequence number in index. - column_name: Column name. - collation: Column sort order ('A' for ascending, NULL for unsorted). - cardinality: Estimated number of unique values. - sub_part: Index prefix length (NULL if full column indexed). - packed: How the key is packed (NULL if not packed). - null: Whether the column allows NULL. - index_type: Index type (BTREE, FULLTEXT, HASH, RTREE). - comment: Index comment. - index_comment: Index comment (ClickHouse 5.5+). - visible: Whether index is visible to optimizer (ClickHouse 8.0+). - expression: Expression for functional index (ClickHouse 8.0+). + table_name: Table name (ClickHouse's column is ``table``). + non_unique: Always ``1``: ClickHouse has no uniqueness constraints to + report, so this is not a discriminator. + key_name: Index name, or ``PRIMARY`` for the sorting-key rows. + seq_in_index: Position within the primary key, or ``1`` for a skip index. + column_name: Column name for a primary-key row — ClickHouse's ``pk_col``. + Empty for a skip-index row, whose expression is in ``expression``. + collation: ``'A'`` ascending, ``'D'`` descending, ``None`` unsorted. + cardinality: Currently always ``0`` per the reference page. + sub_part: Always ``None`` — ClickHouse has no index prefixes. + packed: Always ``None`` — ClickHouse has no packed indexes. + null: Unused by the server. + index_type: ``PRIMARY``, ``MINMAX``, ``SET``, ``BLOOM_FILTER``, + ``TEXT``, ... — ClickHouse's index kinds, not MySQL's ``BTREE``. + comment: Currently always ``''``. + index_comment: Always ``''`` — ClickHouse indexes cannot carry a comment. + visible: Always ``'YES'``. + expression: Index expression for a skip-index row, ``''`` for a + primary-key row. """ table_name: str @@ -243,10 +271,10 @@ class ShowIndexResult: column_name: Optional[str] = None collation: Optional[str] = None cardinality: Optional[int] = None - sub_part: Optional[int] = None + sub_part: Optional[str] = None packed: Optional[str] = None null: Optional[str] = None - index_type: str = "BTREE" + index_type: Optional[str] = None comment: Optional[str] = None index_comment: Optional[str] = None visible: Optional[str] = None @@ -261,8 +289,12 @@ class ShowTableResult: """Result from SHOW TABLES command. Attributes: - name: Table name. - table_type: Table type (BASE TABLE or VIEW), available in FULL mode. + name: Table name. ClickHouse names this column ``name`` in both plain + and ``FULL`` modes (MySQL names it after the database). + table_type: Value of the second column ``SHOW FULL TABLES`` reports. + On ClickHouse that is the **storage engine** (``MergeTree``, + ``ReplicatedMergeTree``, ``View``, ...), not MySQL's ``BASE TABLE`` + / ``VIEW``; ``None`` in plain mode, which reports one column. """ name: str @@ -274,7 +306,8 @@ class ShowDatabaseResult: """Result from SHOW DATABASES command. Attributes: - name: Database name. + name: Database name. ClickHouse names this column ``name``; MySQL + calls it ``Database``. """ name: str @@ -287,9 +320,13 @@ class ShowDatabaseResult: class ShowTriggerResult: """Result from SHOW TRIGGERS command. - NOTE: MySQL-only command, not supported by ClickHouse. - - Contains trigger information. + ClickHouse has no triggers at all: ``SHOW TRIGGERS`` is a ``SYNTAX_ERROR`` + on 26.7.3.19 and the server's own grammar after ``SHOW`` offers no + ``TRIGGERS``. The statement is not emitted — ``format_show_triggers`` + raises ``UnsupportedFeatureError`` with "ClickHouse does not support + triggers" — so the refusal stays the negotiation point for a caller moving + between backends. ``CREATE FUNCTION ... AS`` (an SQL UDF) plus a MATERIALIZED + VIEW is the ClickHouse way to react to a row. Attributes: trigger_name: Trigger name. @@ -318,75 +355,6 @@ class ShowTriggerResult: database_collation: Optional[str] = None -# ==================== Procedure and Function Results ==================== - - -@dataclass -class ShowProcedureResult: - """Result from SHOW PROCEDURE STATUS command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Attributes: - db: Database name. - name: Procedure name. - type: Always 'PROCEDURE'. - definer: Definer of the procedure. - modified: Last modification time. - created: Creation time. - security_type: Security type (DEFINER or INVOKER). - comment: Procedure comment. - character_set_client: Client character set. - collation_connection: Connection collation. - database_collation: Database collation. - """ - - db: str - name: str - type: str = "PROCEDURE" - definer: Optional[str] = None - modified: Optional[str] = None - created: Optional[str] = None - security_type: Optional[str] = None - comment: Optional[str] = None - character_set_client: Optional[str] = None - collation_connection: Optional[str] = None - database_collation: Optional[str] = None - - -@dataclass -class ShowFunctionResult: - """Result from SHOW FUNCTION STATUS command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Attributes: - db: Database name. - name: Function name. - type: Always 'FUNCTION'. - definer: Definer of the function. - modified: Last modification time. - created: Creation time. - security_type: Security type (DEFINER or INVOKER). - comment: Function comment. - character_set_client: Client character set. - collation_connection: Connection collation. - database_collation: Database collation. - """ - - db: str - name: str - type: str = "FUNCTION" - definer: Optional[str] = None - modified: Optional[str] = None - created: Optional[str] = None - security_type: Optional[str] = None - comment: Optional[str] = None - character_set_client: Optional[str] = None - collation_connection: Optional[str] = None - database_collation: Optional[str] = None - - # ==================== Variables and Status Results ==================== @@ -394,7 +362,12 @@ class ShowFunctionResult: class ShowVariableResult: """Result from SHOW VARIABLES command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse. ``SHOW VARIABLES`` and + ``SHOW GLOBAL VARIABLES`` are both ``SYNTAX_ERROR`` on 26.7.3.19 — the + server's grammar after ``SHOW`` offers no ``VARIABLES``. The configuration + lives in ``system.settings`` (what a query or session can change) and + ``system.server_settings`` (the ``config.xml`` half), which is where + ``ClickHouseStatusIntrospector.list_configuration()`` reads it from. Attributes: variable_name: Variable name. @@ -409,7 +382,12 @@ class ShowVariableResult: class ShowStatusResult: """Result from SHOW STATUS command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse. ``SHOW STATUS`` and + ``SHOW GLOBAL STATUS`` are both ``SYNTAX_ERROR`` on 26.7.3.19. ClickHouse + splits the same idea three ways: ``system.metrics`` (instantaneous gauges), + ``system.events`` (monotonic counters) and ``system.asynchronous_metrics`` + (sampled OS / disk figures), which is exactly what + ``ClickHouseStatusIntrospector.list_performance_metrics()`` reads. Attributes: variable_name: Status variable name. @@ -427,7 +405,9 @@ class ShowStatusResult: class ShowWarningResult: """Result from SHOW WARNINGS command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse. ``SHOW WARNINGS`` is + a ``SYNTAX_ERROR`` on 26.7.3.19; the server reports a problem by raising it, + and once configured records it in ``system.text_log`` / ``system.query_log``. Attributes: level: Warning level (Note, Warning, Error). @@ -440,19 +420,6 @@ class ShowWarningResult: message: str -@dataclass -class ShowCountResult: - """Result from SHOW COUNT(*) WARNINGS/ERRORS command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Attributes: - count: Number of warnings or errors. - """ - - count: int - - # ==================== Grants Results ==================== @@ -460,10 +427,20 @@ class ShowCountResult: class ShowGrantResult: """Result from SHOW GRANTS command. - NOTE: MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement — ``SHOW GRANTS [FOR user1 [, user2 ...]] + [WITH IMPLICIT] [FINAL]`` — and returns one row per grant. + + The column it returns has no stable name: the server labels it after the + statement text plus the output format the client asked for. Measured on + 26.7.3.19: ``SHOW GRANTS`` over HTTP with ``default_format=JSONEachRow`` + gives ``GRANTS``; ``SHOW GRANTS FORMAT JSONEachRow`` gives ``GRANTS FORMAT + JSONEachRow``; over the native protocol, where clickhouse-connect appends + ``FORMAT Native``, ``GRANTS FORMAT Native``; and ``SHOW GRANTS FOR root`` + gives ``GRANTS FOR root``. MySQL's ``Grants for`` never appears. The parser + therefore reads the single value of each row rather than a named column. Attributes: - grants: List of GRANT statements for the user. + grants: One GRANT statement. """ grants: str @@ -476,51 +453,52 @@ class ShowGrantResult: class ShowProcessListResult: """Result from SHOW PROCESSLIST command. - NOTE: MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement and it is ``system.processes`` verbatim, so its + columns are that table's 43 lower-cased names: ``is_initial_query``, ``user``, + ``query_id``, ``address``, ``port``, ``initial_user``, ``initial_query_id``, + ``initial_address``, ``initial_port``, ``interface``, ``os_user``, + ``client_hostname``, ``client_name``, ``client_agent``, ``client_revision``, + ``client_version_major``, ``client_version_minor``, ``client_version_patch``, + ``http_method``, ``http_user_agent``, ``http_referer``, ``forwarded_for``, + ``quota_key``, ``distributed_depth``, ``elapsed``, ``is_cancelled``, + ``is_all_data_sent``, ``read_rows``, ``read_bytes``, ``total_rows_approx``, + ``written_rows``, ``written_bytes``, ``memory_usage``, + ``peak_memory_usage``, ``query``, ``normalized_query_hash``, ``query_kind``, + ``thread_ids``, ``peak_threads_usage``, ``ProfileEvents``, ``Settings``, + ``current_database``, ``is_internal`` (26.7.3.19). + + The MySQL-shaped field names below are kept so the result reads the same + across backends, but two of them have no ClickHouse source and stay ``None``; + see :attr:`command` and :attr:`state`. There is no ``FULL`` variant of the + statement either, and none is needed — ``query`` already holds the full + query text. Attributes: - id: Connection identifier. - user: ClickHouse user. - host: Client host and port. - command: Command type (Query, Sleep, etc.). - time: Time in current state (seconds). - db: Selected database. - state: Thread state. - info: Query text (NULL if not executing query, or in basic mode). + id: ``query_id`` — a UUID identifying the running query, **not** MySQL's + integer connection id, which ClickHouse does not report here. + user: The ClickHouse user the query runs as (``user``). + host: ``address`` and ``port`` joined as ``host:port``, matching the + shape of MySQL's ``Host`` column. The driver may hand back ``address`` + as an ``IPv6Address``; it is stringified here. + command: Always ``None``. ``system.processes`` has no per-process command + column, because every row *is* a running query. + time: ``elapsed`` — seconds since the query started, as a float. + db: ``current_database``. + state: Always ``None``. There is no per-process state column; the closest + server-side signal is ``is_cancelled``. + info: ``query`` — the query text. """ - id: int + id: Optional[str] user: str - host: str - command: str - time: int + host: Optional[str] + command: Optional[str] = None + time: Optional[float] = None db: Optional[str] = None state: Optional[str] = None info: Optional[str] = None -# ==================== Open Tables Results ==================== - - -@dataclass -class ShowOpenTableResult: - """Result from SHOW OPEN TABLES command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Attributes: - database: Database name. - table_name: Table name. - in_use: Number of table locks in use. - name_locked: Whether table name is locked. - """ - - database: str - table_name: str - in_use: int - name_locked: int - - # ==================== Engine Results ==================== @@ -528,21 +506,51 @@ class ShowOpenTableResult: class ShowEngineResult: """Result from SHOW ENGINES command. - NOTE: MySQL-only command, not supported by ClickHouse. + ClickHouse has this statement and it is ``system.table_engines`` verbatim. + Its columns are its own, and MySQL's ``Support`` / ``Transactions`` / ``XA`` / + ``Savepoints`` are not among them — this result type therefore carries the + eight ``supports_*`` capability flags ClickHouse actually reports, plus the + engine's own documentation columns. On 26.7.3.19 the statement returns 83 + rows, one per engine. + + ``introduced_in`` is the only version-shaped column in the whole ``SHOW`` + set — it is where a table engine's introducing release would come from. It is + **empty on this server**: ``SELECT count() FROM system.table_engines WHERE + introduced_in != ''`` returns 0 across all 83 engines, so no version is + claimed from it here. Attributes: - engine: Storage engine name. - support: Support level (YES, NO, DEFAULT). - transactions: Whether engine supports transactions. - xa: Whether engine supports XA transactions. - savepoints: Whether engine supports savepoints. + engine: Engine name (ClickHouse's column is ``name``). + supports_settings: Whether the engine accepts a ``SETTINGS`` clause. + supports_skipping_indices: Whether it supports data skipping indices. + supports_projections: Whether it supports projections. + supports_sort_order: Whether it supports an explicit sorting key. + supports_ttl: Whether it supports TTL. + supports_replication: Whether it supports replication. + supports_deduplication: Whether it supports block deduplication. + supports_parallel_insert: Whether it supports parallel insert. + description: Markdown description of the engine. + syntax: Markdown syntax block. + examples: Markdown examples block. + introduced_in: Release the engine was introduced in — ``''`` on + 26.7.3.19, where the server populates none of them. + related: Array of related engine names. """ engine: str - support: str - transactions: Optional[str] = None - xa: Optional[str] = None - savepoints: Optional[str] = None + supports_settings: Optional[int] = None + supports_skipping_indices: Optional[int] = None + supports_projections: Optional[int] = None + supports_sort_order: Optional[int] = None + supports_ttl: Optional[int] = None + supports_replication: Optional[int] = None + supports_deduplication: Optional[int] = None + supports_parallel_insert: Optional[int] = None + description: Optional[str] = None + syntax: Optional[str] = None + examples: Optional[str] = None + introduced_in: Optional[str] = None + related: Optional[Any] = None # ==================== Charset and Collation Results ==================== @@ -552,7 +560,9 @@ class ShowEngineResult: class ShowCharsetResult: """Result from SHOW CHARACTER SET command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse. ``SHOW CHARACTER SET`` + is a ``SYNTAX_ERROR`` on 26.7.3.19; the equivalents are the + ``system.character_sets`` table and the ``output_format_*`` settings. Attributes: charset: Character set name. @@ -571,7 +581,10 @@ class ShowCharsetResult: class ShowCollationResult: """Result from SHOW COLLATION command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse. ``SHOW COLLATION`` is a + ``SYNTAX_ERROR`` on 26.7.3.19; the equivalents are the ``system.collations`` + table and, per query, ``ORDER BY`` on ``String`` (which is a byte-wise + comparison, not a linguistic one). Attributes: collation: Collation name. @@ -597,7 +610,10 @@ class ShowCollationResult: class ShowPluginResult: """Result from SHOW PLUGINS command. - NOTE: MySQL-only command, not supported by ClickHouse. + NOTE: MySQL-only command, not supported by ClickHouse, which has no plugin + concept at all — ``SHOW PLUGINS`` is a ``SYNTAX_ERROR`` on 26.7.3.19 and the + server's grammar after ``SHOW`` offers no ``PLUGINS``. The nearest things + that exist are table engines, SQL UDFs, dictionaries and named collections. Attributes: name: Plugin name. @@ -612,70 +628,3 @@ class ShowPluginResult: type: str library: Optional[str] = None license: Optional[str] = None - - -# ==================== Profile Results (Deprecated) ==================== - - -@dataclass -class ShowProfileResult: - """Result from SHOW PROFILE command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Note: Deprecated in ClickHouse 5.7+, use Performance Schema instead. - - Attributes: - status: Profile status. - duration: Duration in seconds. - cpu_user: CPU user time (if CPU profile). - cpu_system: CPU system time (if CPU profile). - context_voluntary: Voluntary context switches (if BLOCK IO profile). - context_involuntary: Involuntary context switches (if BLOCK IO profile). - block_ops_in: Block operations in (if BLOCK IO profile). - block_ops_out: Block operations out (if BLOCK IO profile). - messages_sent: Messages sent (if IPC profile). - messages_received: Messages received (if IPC profile). - page_faults_major: Major page faults (if PAGE FAULTS profile). - page_faults_minor: Minor page faults (if PAGE FAULTS profile). - swaps: Number of swaps (if PAGE FAULTS profile). - source_function: Source function name (if SOURCE profile). - source_file: Source file name (if SOURCE profile). - source_line: Source line number (if SOURCE profile). - """ - - status: str - duration: float - cpu_user: Optional[float] = None - cpu_system: Optional[float] = None - context_voluntary: Optional[int] = None - context_involuntary: Optional[int] = None - block_ops_in: Optional[int] = None - block_ops_out: Optional[int] = None - messages_sent: Optional[int] = None - messages_received: Optional[int] = None - page_faults_major: Optional[int] = None - page_faults_minor: Optional[int] = None - swaps: Optional[int] = None - source_function: Optional[str] = None - source_file: Optional[str] = None - source_line: Optional[int] = None - - -@dataclass -class ShowProfilesResult: - """Result from SHOW PROFILES command. - - NOTE: MySQL-only command, not supported by ClickHouse. - - Note: Deprecated in ClickHouse 5.7+, use Performance Schema instead. - - Attributes: - query_id: Query identifier. - duration: Query duration in seconds. - query: Query text. - """ - - query_id: int - duration: float - query: str diff --git a/tests/conftest.py b/tests/conftest.py index c5151cb..919628c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,9 +40,11 @@ def setup_asyncio_broken_pipe_handler(): """ Set up asyncio event loop exception handler to suppress BrokenPipeError. - In ClickHouse 5.6 + Python 3.8 asyncio combination, writes to dead connections - may raise BrokenPipeError through the asyncio transport layer via the - event loop's exception handler rather than through normal try/except. + Writing to a connection the server has already dropped raises + BrokenPipeError, and under asyncio that arrives through the event loop's + exception handler rather than through a normal try/except around the write. + The exception comes from the socket layer, not from a server release, so it + is suppressed regardless of which ClickHouse version is connected. This fixture sets up the handler at session start and restores it at end. """ diff --git a/tests/providers/basic.py b/tests/providers/basic.py index 8103e56..5480c07 100644 --- a/tests/providers/basic.py +++ b/tests/providers/basic.py @@ -348,11 +348,15 @@ def get_dialect(self, scenario_name: str = "default"): """Return a bare, fully-constructed ClickHouse dialect instance. Used by the ``feature/basic/ddl`` subtopic (expression/dialect - contract). ``(8, 0, 0)`` is the version the ddl provider used. + contract). A version has to be supplied because an unadapted + ``ClickHouseDialect()`` raises ``DialectNotAdaptedException`` the moment + ``dialect.version`` is read; ``(26, 7, 1)`` is the current stable line + and is the same version the backend's own dialect tests use. It used to + be ``(8, 0, 0)``, which is not a ClickHouse release. """ from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect - return ClickHouseDialect(version=(8, 0, 0)) + return ClickHouseDialect(version=(26, 7, 1)) def _track_backend(self, backend_instance, collection: List) -> None: if backend_instance not in collection: diff --git a/tests/providers/partition.py b/tests/providers/partition.py deleted file mode 100644 index 46ace6b..0000000 --- a/tests/providers/partition.py +++ /dev/null @@ -1,260 +0,0 @@ -"""ClickHouse provider for shared partition testsuite scenarios.""" - -from __future__ import annotations - -from datetime import datetime -from decimal import Decimal -from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type - -from rhosocial.activerecord.base.field_proxy import FieldProxy -from rhosocial.activerecord.backend.impl.clickhouse.backend.async_backend import AsyncClickHouseBackend -from rhosocial.activerecord.model import ActiveRecord, AsyncActiveRecord -from rhosocial.activerecord.testsuite.feature.partition.interfaces import IPartitionProvider - -from .scenarios import get_enabled_scenarios, get_scenario - - -class PartitionProvider(IPartitionProvider): - """Concrete partition provider backed by real ClickHouse scenarios.""" - - TABLE_NAME = "ar_testsuite_partition_events" - PARTITIONS = { - "p2026_01": ("p2026_01", "2026-02-01"), - "p2026_02": ("p2026_02", "2026-03-01"), - "p2026_03": ("p2026_03", "2026-04-01"), - } - - def __init__(self): - self._active_backends = [] - self._active_async_backends = [] - - def get_test_scenarios(self) -> List[str]: - """Return ClickHouse scenarios configured for real backend tests.""" - return list(get_enabled_scenarios().keys()) - - def get_partition_capabilities(self, scenario_name: str) -> Dict[str, bool]: - backend = self._ensure_backend(scenario_name) - return self._capabilities(backend) - - async def async_get_partition_capabilities(self, scenario_name: str) -> Dict[str, bool]: - backend = await self._ensure_async_backend(scenario_name) - return self._capabilities(backend) - - def setup_range_partitioned_event_fixtures( - self, - scenario_name: str, - ) -> Tuple[Type[ActiveRecord], ...]: - backend = self._ensure_backend(scenario_name) - self._reset_partition_table(backend) - return (self._event_model(backend),) - - async def setup_async_range_partitioned_event_fixtures( - self, - scenario_name: str, - ) -> Tuple[Type[ActiveRecord], ...]: - backend = await self._ensure_async_backend(scenario_name) - await self._reset_partition_table_async(backend) - return (self._async_event_model(backend),) - - def add_future_range_partition(self, scenario_name: str) -> None: - backend = self._ensure_backend(scenario_name) - backend.execute( - f"ALTER TABLE `{self.TABLE_NAME}` ADD PARTITION (PARTITION p2026_03 VALUES LESS THAN ('2026-04-01'))" - ) - - async def async_add_future_range_partition(self, scenario_name: str) -> None: - backend = await self._ensure_async_backend(scenario_name) - await backend.execute( - f"ALTER TABLE `{self.TABLE_NAME}` ADD PARTITION (PARTITION p2026_03 VALUES LESS THAN ('2026-04-01'))" - ) - - def truncate_partition(self, scenario_name: str, partition_key: str) -> None: - backend = self._ensure_backend(scenario_name) - backend.execute(f"ALTER TABLE `{self.TABLE_NAME}` TRUNCATE PARTITION {partition_key}") - - async def async_truncate_partition(self, scenario_name: str, partition_key: str) -> None: - backend = await self._ensure_async_backend(scenario_name) - await backend.execute(f"ALTER TABLE `{self.TABLE_NAME}` TRUNCATE PARTITION {partition_key}") - - def detach_partition(self, scenario_name: str, partition_key: str) -> None: - raise NotImplementedError("ClickHouse does not support DETACH PARTITION") - - async def async_detach_partition(self, scenario_name: str, partition_key: str) -> None: - raise NotImplementedError("ClickHouse does not support DETACH PARTITION") - - def attach_partition(self, scenario_name: str, partition_key: str) -> None: - raise NotImplementedError("ClickHouse does not support ATTACH PARTITION") - - async def async_attach_partition(self, scenario_name: str, partition_key: str) -> None: - raise NotImplementedError("ClickHouse does not support ATTACH PARTITION") - - def get_partition_metadata(self, scenario_name: str) -> Dict[str, Any]: - backend = self._ensure_backend(scenario_name) - rows = backend.fetch_all(self._metadata_sql(), (self.TABLE_NAME,)) - return self._metadata_dict(rows) - - async def async_get_partition_metadata(self, scenario_name: str) -> Dict[str, Any]: - backend = await self._ensure_async_backend(scenario_name) - rows = await backend.fetch_all(self._metadata_sql(), (self.TABLE_NAME,)) - return self._metadata_dict(rows) - - def create_valid_unique_constraint(self, scenario_name: str) -> None: - backend = self._ensure_backend(scenario_name) - backend.execute( - f"CREATE UNIQUE INDEX `{self.TABLE_NAME}_id_created_at_uq` ON `{self.TABLE_NAME}` (id, created_at)" - ) - - async def async_create_valid_unique_constraint(self, scenario_name: str) -> None: - backend = await self._ensure_async_backend(scenario_name) - await backend.execute( - f"CREATE UNIQUE INDEX `{self.TABLE_NAME}_id_created_at_uq` ON `{self.TABLE_NAME}` (id, created_at)" - ) - - def create_invalid_unique_constraint(self, scenario_name: str) -> None: - backend = self._ensure_backend(scenario_name) - backend.execute(f"CREATE UNIQUE INDEX `{self.TABLE_NAME}_id_only_uq` ON `{self.TABLE_NAME}` (id)") - - async def async_create_invalid_unique_constraint(self, scenario_name: str) -> None: - backend = await self._ensure_async_backend(scenario_name) - await backend.execute(f"CREATE UNIQUE INDEX `{self.TABLE_NAME}_id_only_uq` ON `{self.TABLE_NAME}` (id)") - - def cleanup_after_test(self, scenario_name: str) -> None: - for backend in self._active_backends: - try: - self._drop_partition_table(backend) - finally: - backend.disconnect() - self._active_backends.clear() - - async def cleanup_after_test_async(self, scenario_name: str) -> None: - for backend in self._active_async_backends: - try: - await self._drop_partition_table_async(backend) - finally: - await backend.disconnect() - self._active_async_backends.clear() - - def _ensure_backend(self, scenario_name: str): - if self._active_backends: - return self._active_backends[0] - backend_class, config = get_scenario(scenario_name) - backend = backend_class(connection_config=config) - backend.connect() - backend.introspect_and_adapt() - self._active_backends.append(backend) - return backend - - async def _ensure_async_backend(self, scenario_name: str): - if self._active_async_backends: - return self._active_async_backends[0] - _, config = get_scenario(scenario_name) - backend = AsyncClickHouseBackend(connection_config=config) - await backend.connect() - await backend.introspect_and_adapt() - self._active_async_backends.append(backend) - return backend - - def _capabilities(self, backend) -> Dict[str, bool]: - return { - "range_partitioning": backend.dialect.supports_range_table_partitioning(), - "range_columns_partitioning": backend.dialect.supports_range_columns_partitioning(), - "list_partitioning": backend.dialect.supports_list_table_partitioning(), - "hash_partitioning": backend.dialect.supports_hash_table_partitioning(), - "key_partitioning": backend.dialect.supports_key_table_partitioning(), - "subpartitioning": backend.dialect.supports_subpartitioning(), - "add_partition": backend.dialect.supports_add_partition(), - "truncate_partition": backend.dialect.supports_truncate_partition(), - "detach_partition": backend.dialect.supports_detach_partition(), - "attach_partition": backend.dialect.supports_attach_partition(), - "partition_introspection": True, - "partition_metadata": backend.dialect.supports_partition_metadata_introspection(), - "partition_explain": backend.dialect.supports_explain_format("TEXT"), - "partition_pruning_explain": backend.dialect.supports_explain_format("TEXT"), - "partition_bounds": True, - "partitioned_unique_constraint": True, - "unique_requires_partition_key": True, - } - - def _reset_partition_table(self, backend) -> None: - self._drop_partition_table(backend) - backend.execute(self._create_table_sql()) - - async def _reset_partition_table_async(self, backend) -> None: - await self._drop_partition_table_async(backend) - await backend.execute(self._create_table_sql()) - - def _drop_partition_table(self, backend) -> None: - backend.execute(f"DROP TABLE IF EXISTS `{self.TABLE_NAME}`") - - async def _drop_partition_table_async(self, backend) -> None: - await backend.execute(f"DROP TABLE IF EXISTS `{self.TABLE_NAME}`") - - def _create_table_sql(self) -> str: - return f""" - CREATE TABLE `{self.TABLE_NAME}` ( - id BIGINT NOT NULL AUTO_INCREMENT, - created_at DATETIME NOT NULL, - tenant_id INT NOT NULL, - payload TEXT, - amount DECIMAL(12, 2), - KEY idx_created_at (created_at), - KEY idx_id (id) - ) - PARTITION BY RANGE COLUMNS (created_at) ( - PARTITION p2026_01 VALUES LESS THAN ('2026-02-01'), - PARTITION p2026_02 VALUES LESS THAN ('2026-03-01') - ) - """ - - def _metadata_sql(self) -> str: - return """ - SELECT PARTITION_NAME AS name, - PARTITION_METHOD AS strategy, - PARTITION_EXPRESSION AS partition_key, - PARTITION_DESCRIPTION AS bound - FROM information_schema.PARTITIONS - WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = %s - AND PARTITION_NAME IS NOT NULL - ORDER BY PARTITION_NAME - """ - - def _metadata_dict(self, rows) -> Dict[str, Any]: - return { - "is_partitioned": bool(rows), - "strategy": "range", - "key_columns": ["created_at"], - "partitions": [{"name": row["name"], "bound": str(row["bound"])} for row in rows], - } - - def _event_model(self, backend): - table_name = self.TABLE_NAME - - class PartitionEvent(ActiveRecord): - __table_name__ = table_name - __backend__ = backend - c: ClassVar[FieldProxy] = FieldProxy() - - id: Optional[int] = None - created_at: datetime - tenant_id: int - payload: Optional[str] = None - amount: Optional[Decimal] = None - - return PartitionEvent - - def _async_event_model(self, backend): - table_name = self.TABLE_NAME - - class AsyncPartitionEvent(AsyncActiveRecord): - __table_name__ = table_name - __backend__ = backend - c: ClassVar[FieldProxy] = FieldProxy() - - id: Optional[int] = None - created_at: datetime - tenant_id: int - payload: Optional[str] = None - amount: Optional[Decimal] = None - - return AsyncPartitionEvent diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_adapters_table_mixins.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_adapters_table_mixins.py index 0fc6ef4..51cb800 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_adapters_table_mixins.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_adapters_table_mixins.py @@ -20,7 +20,6 @@ ClickHouseJSONAdapter, ClickHouseTimeAdapter, ClickHouseUUIDAdapter, - ClickHouseVectorAdapter, ) from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect from rhosocial.activerecord.backend.impl.clickhouse.mixins.ddl_table import ClickHouseTableMixin @@ -109,12 +108,25 @@ def test_datetime_adapter(self): assert parsed.year == 2024 and parsed.hour == 10 assert a.from_database(None, datetime) is None - def test_vector_adapter(self): - a = ClickHouseVectorAdapter() - assert a.to_database([1.0, 2.0], list) == "[1.0,2.0]" - assert a.to_database(None, list) is None - assert a.from_database("[1.0, 2.0]", list) == [1.0, 2.0] - assert a.from_database(None, list) is None + def test_there_is_no_vector_adapter(self): + """``ClickHouseVectorAdapter`` adapted MySQL 9.0's ``VECTOR`` column. + + ClickHouse has no such type — ``CREATE TABLE t (a VECTOR(4))`` is + rejected with ``Unknown data type family: VECTOR`` on 26.7.3.19 — and the + adapter's only real content was MySQL's 16384-dimension ceiling. What + ClickHouse stores an embedding in is ``Array(Float32)``, which is a plain + Python ``list`` of floats and needs no adapter: the driver already returns + an array column as a list (``test_array_roundtrip`` in + ``types/test_native_types.py``). + + Asserted as an absence, so re-adding the class does not quietly restore a + ``VECTOR(n)`` adapter. + """ + import rhosocial.activerecord.backend.impl.clickhouse.adapters as adapters + + assert "ClickHouseVectorAdapter" not in vars(adapters) + assert not [name for name in vars(adapters) if "Vector" in name] + assert not hasattr(adapters, "MAX_VECTOR_DIMENSION") class TestEnumAdapter: diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_column_mapping.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_column_mapping.py index 371e1b7..f686a03 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_column_mapping.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/adapters/test_column_mapping.py @@ -31,7 +31,13 @@ def setup_mapped_users_table(clickhouse_backend): def test_insert_with_mapping(clickhouse_backend, setup_mapped_users_table): """ Tests that execute() with an INSERT correctly handles mapped data. - Note: ClickHouse < 8.0.1 does not support RETURNING, so we verify with a subsequent SELECT. + + ClickHouse has no ``RETURNING`` clause at any version — the ``INSERT INTO`` + reference page's syntax block is ``INSERT INTO [TABLE] [db.]table [(c1, c2, + c3)] [SETTINGS ...] VALUES ...`` and ``RETURNING`` appears nowhere in it — so + the inserted row is verified with a subsequent ``SELECT``. This note used to + say "ClickHouse < 8.0.1", a version gate that no ClickHouse server could be + checked against. """ backend = clickhouse_backend now = datetime.now() diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/conftest.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/conftest.py index 029cb60..b7daa1f 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/conftest.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/conftest.py @@ -323,9 +323,19 @@ def check_protocol_requirements(request): @pytest.fixture(scope="function") def clickhouse_dialect(): - """Fixture providing ClickHouseDialect instance for testing transaction expressions.""" + """Fixture providing ClickHouseDialect instance for testing transaction expressions. + + The version has to be set: ``ClickHouseDialect()`` with no version raises + ``DialectNotAdaptedException`` from the moment anything reads + ``dialect.version``. It is set to ``(26, 7, 1)`` — a real ClickHouse release + line, and the same one ``test_unsupported_features.py`` uses — because the + value is not cosmetic: ``dialect.version`` is what the real gates compare + against, including ``supports_explain_analyze()`` at ``>= (26, 7, 0)``, the + JSON gates at ``>= (26, 0, 0)``, the function-version table and the ``QBit`` + floor. The fabricated ``(8, 0, 0)`` this used to set compares *below* every + one of them, so it silently reported the server as too old for features it + has. ClickHouse's numbering has never had an 8.0. + """ from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect - dialect = ClickHouseDialect() - dialect.version = (8, 0, 0) - return dialect + return ClickHouseDialect(version=(26, 7, 1)) diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_coverage.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_coverage.py index 8dd2a39..ea55f8f 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_coverage.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_coverage.py @@ -6,7 +6,7 @@ - RENAME TABLE (atomic multi-table rename) - TRUNCATE TABLE (option guards) -- ALTER TABLE ... ALTER COLUMN {SET DEFAULT | DROP DEFAULT} +- ALTER TABLE ... ALTER COLUMN (a default change, in ClickHouse's own spelling) MySQL-only statement families (whole-table ``ANALYZE``/``CHECK``/``CHECKSUM``/ ``REPAIR`` maintenance, stored ``PROCEDURE``/``FUNCTION``/``CALL``, ``TABLE``/ @@ -83,21 +83,71 @@ def test_cascade_unsupported(self, dialect): class TestAlterColumnDefault: - """Test ALTER TABLE ... ALTER COLUMN {SET DEFAULT | DROP DEFAULT}.""" + """``SET DEFAULT`` / ``DROP DEFAULT`` render as ClickHouse's own spellings. + + ``ALTER COLUMN c SET DEFAULT v`` is a ``SYNTAX_ERROR`` on 26.7.3.19 — the + server's expected-token list after ``ALTER COLUMN c`` contains ``DEFAULT`` + and not ``SET`` — and ``ALTER COLUMN c DROP DEFAULT`` is a ``SYNTAX_ERROR`` + too, because removing a column property is + ``MODIFY COLUMN c REMOVE ``. See + ``https://clickhouse.com/docs/reference/statements/alter/column``. + """ def test_set_default_string(self, dialect): action = AlterColumn(dialect, "col", ColumnAlterOperation.SET_DEFAULT, new_value="ABC") sql, params = AlterTableExpression(dialect, Table(dialect, "t"), [action]).to_sql() - assert "ALTER COLUMN `col` SET DEFAULT 'ABC'" in sql + assert "ALTER COLUMN `col` DEFAULT 'ABC'" in sql assert params == () def test_set_default_integer(self, dialect): action = AlterColumn(dialect, "num", ColumnAlterOperation.SET_DEFAULT, new_value=5) sql, _ = AlterTableExpression(dialect, Table(dialect, "t"), [action]).to_sql() - assert "ALTER COLUMN `num` SET DEFAULT 5" in sql + assert "ALTER COLUMN `num` DEFAULT 5" in sql - def test_drop_default(self, dialect): + def test_drop_default_uses_modify_remove(self, dialect): action = AlterColumn(dialect, "col", ColumnAlterOperation.DROP_DEFAULT) sql, params = AlterTableExpression(dialect, Table(dialect, "t"), [action]).to_sql() - assert "ALTER COLUMN `col` DROP DEFAULT" in sql + assert "MODIFY COLUMN `col` REMOVE DEFAULT" in sql assert params == () + + def test_no_set_or_drop_default_keyword_is_rendered(self, dialect): + """The two spellings the server rejects must not appear in the output.""" + set_sql, _ = AlterTableExpression( + dialect, Table(dialect, "t"), [AlterColumn(dialect, "col", ColumnAlterOperation.SET_DEFAULT, new_value=1)] + ).to_sql() + drop_sql, _ = AlterTableExpression( + dialect, Table(dialect, "t"), [AlterColumn(dialect, "col", ColumnAlterOperation.DROP_DEFAULT)] + ).to_sql() + assert "SET DEFAULT" not in set_sql + assert "DROP DEFAULT" not in drop_sql + + def test_both_renderings_execute_on_the_server(self, clickhouse_backend_single): + """The rendered SQL is what the server accepts, and it changes the metadata. + + Before this was corrected the dialect emitted ``ALTER COLUMN c SET + DEFAULT 5`` and ``ALTER COLUMN c DROP DEFAULT``, both of which 26.7.3.19 + rejects with ``Code: 62 ... Expected one of: token sequence, Dot, token, + REMOVE, MODIFY SETTING, RESET SETTING, ADD ENUM VALUES, NULL, NOT, + DEFAULT, MATERIALIZED, EPHEMERAL, ALIAS, AUTO_INCREMENT, TTL, PRIMARY + KEY, COMMENT, CODEC, TYPE. (SYNTAX_ERROR)``. + """ + backend = clickhouse_backend_single + backend.execute("DROP TABLE IF EXISTS ar_alter_default") + backend.execute("CREATE TABLE ar_alter_default (id UInt32) ENGINE = MergeTree ORDER BY id") + try: + d = backend.dialect + backend.execute(*AlterTableExpression( + d, Table(d, "ar_alter_default"), + [AlterColumn(d, "id", ColumnAlterOperation.SET_DEFAULT, new_value=7)], + ).to_sql()) + create = backend.execute("SHOW CREATE TABLE ar_alter_default").data[0]["statement"] + assert "DEFAULT 7" in create + + backend.execute(*AlterTableExpression( + d, Table(d, "ar_alter_default"), + [AlterColumn(d, "id", ColumnAlterOperation.DROP_DEFAULT)], + ).to_sql()) + create = backend.execute("SHOW CREATE TABLE ar_alter_default").data[0]["statement"] + assert "DEFAULT" not in create + finally: + backend.execute("DROP TABLE IF EXISTS ar_alter_default") diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_source_mixin.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_source_mixin.py index 782df0e..08deb9e 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_source_mixin.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_ddl_source_mixin.py @@ -9,7 +9,8 @@ import pytest -from rhosocial.activerecord.backend.expression.core import Column +from rhosocial.activerecord.backend.expression.core import Column, Literal +from rhosocial.activerecord.backend.expression.predicates import ComparisonPredicate from rhosocial.activerecord.backend.expression.statements import ( ColumnConstraintType, CreateTableExpression, @@ -59,7 +60,9 @@ ColumnConstraintType.UNIQUE, name="uq_value", ) -CHECK_CONDITION = Column(None, "physical_value") == "active" +CHECK_CONDITION = ComparisonPredicate( + None, "=", Column(None, "physical_value"), Literal(None, "active") +) CHECK_CONSTRAINT = UseConstraint( ColumnConstraintType.CHECK, name="ck_value", @@ -107,7 +110,9 @@ def generated_expression(dialect): None, TableConstraintType.CHECK, name="ck_physical_quantity", - check_condition=Column(None, "physical_value") == "active", + check_condition=ComparisonPredicate( + None, "=", Column(None, "physical_value"), Literal(None, "active") + ), ) TABLE_CONSTRAINTS = [TABLE_CONSTRAINT_ONE, TABLE_CONSTRAINT_TWO] TABLE_INDEX_ONE = ClickHouseIndexDefinition( @@ -329,11 +334,13 @@ def _assert_declaration_model(model): CLICKHOUSE_OPTIONS[1], ) assert metadata.annotations == expected_annotations - assert model.column_type("value") is SQL_TYPE - assert model.column_type("value").data_types == NATIVE_TYPES + assert model.column_data_type("value") is SQL_TYPE + assert model.column_data_type("value").data_types == NATIVE_TYPES assert all( actual is expected - for actual, expected in zip(model.column_type("value").data_types, NATIVE_TYPES, strict=True) + for actual, expected in zip( + model.column_data_type("value").data_types, NATIVE_TYPES, strict=True + ) ) value_constraints = model.column_constraints("value") @@ -420,9 +427,9 @@ def _assert_batch_interfaces(model): "id": "record_id", "quantity": "quantity", } - assert list(model.columns_type(fields)) == fields - assert model.columns_type(fields)["value"] is SQL_TYPE - assert model.columns_type(fields)["quantity"] is None + assert list(model.columns_data_type(fields)) == fields + assert model.columns_data_type(fields)["value"] is SQL_TYPE + assert model.columns_data_type(fields)["quantity"] is None assert list(model.columns_constraints(fields)) == fields assert model.columns_constraints(fields)["value"][0] is DEFAULT_CONSTRAINT.constraint assert list(model.columns_attributes(fields)) == fields @@ -446,6 +453,7 @@ def _assert_batch_interfaces(model): def _assert_plain_defaults(model): assert isinstance(model, DDLSource) assert model.ddl_field_names() == ("id", "value") + assert model.column_data_type("value") is None assert model.column_type("value") is None assert model.column_constraints("value") == [] assert model.column_attributes("value") == [] @@ -509,7 +517,7 @@ async def test_sync_and_async_declarations_are_identical(): sync_model, async_model = MODEL_CLASSES assert sync_model.ddl_field_names() == async_model.ddl_field_names() assert sync_model.columns_name() == async_model.columns_name() - assert sync_model.column_type("value") is async_model.column_type("value") + assert sync_model.column_data_type("value") is async_model.column_data_type("value") assert sync_model.column_constraints("value") == async_model.column_constraints("value") assert sync_model.column_attributes("value") == async_model.column_attributes("value") sync_indexes = sync_model.column_indexes("value") @@ -543,7 +551,7 @@ def test_clickhouse_column_options_are_collected_with_explicit_handler(): def test_collected_clickhouse_type_and_options_render_minimal_column(): dialect = ClickHouseDialect(version=(26, 7, 3)) - data_type = DeclarationModel.column_type("value").data_types[0] + data_type = DeclarationModel.column_data_type("value").data_types[0] assert isinstance(data_type, ClickHouseStringType) option = DeclarationModel.column_options("value")[0] data_type.dialect = dialect diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_partition_maintenance.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_partition_maintenance.py new file mode 100644 index 0000000..34de8f9 --- /dev/null +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_partition_maintenance.py @@ -0,0 +1,208 @@ +# tests/rhosocial/activerecord_clickhouse_test/feature/backend/ddl/test_partition_maintenance.py +""" +Live-server contract for ClickHouse partition maintenance. + +The point of these tests is that the rendered SQL is **executable**. The +MySQL partition surface this file's module replaced (``ADD PARTITION``, +``TRUNCATE PARTITION``, ``REORGANIZE PARTITION``, ``SUBPARTITION BY``, +``PARTITION ... VALUES LESS THAN``) was the same defect class as the +``VECTOR(4)`` case: statements the backend offered that the server rejects. +Each refusal below is therefore checked against the server too, not only +against the dialect. + +References: +https://clickhouse.com/docs/engines/table-engines/mergetree-family/custom-partitioning-key +https://clickhouse.com/docs/sql-reference/statements/alter/partition +""" + +import pytest + +from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError +from rhosocial.activerecord.backend.expression.core import Column +from rhosocial.activerecord.backend.expression.statements.ddl_partition import ( + PartitionClause, + PartitionDefinition, + PartitionStrategy, +) +from rhosocial.activerecord.backend.impl.clickhouse.expression import ( + ClickHouseAttachPartitionExpression, + ClickHouseDetachPartitionExpression, + ClickHouseDropPartitionExpression, +) + +TABLE = "ar_clickhouse_partition_events" + + +@pytest.fixture(scope="module") +def dialect(): + from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect + + return ClickHouseDialect(version=(26, 7, 3)) + + +@pytest.fixture +def partitioned_backend(clickhouse_backend_single): + """Backend on a MergeTree table partitioned by ``toYYYYMM(created_at)``. + + ``toYYYYMM`` is how ClickHouse partitions by month: the partition key is an + arbitrary expression, and the resulting partition ids are ``YYYYMM`` + strings. That is the whole of ClickHouse's partitioning model — there is no + strategy keyword and no boundary list to declare. + """ + backend = clickhouse_backend_single + backend.execute(f"DROP TABLE IF EXISTS {TABLE}") + backend.execute( + f"CREATE TABLE {TABLE} (id UInt32, created_at DateTime) " + "ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY id" + ) + backend.execute( + f"INSERT INTO {TABLE} (id, created_at) VALUES (1, '2026-01-05'), (2, '2026-01-06')" + ) + backend.execute( + f"INSERT INTO {TABLE} (id, created_at) VALUES (3, '2026-02-05')" + ) + yield backend + backend.execute(f"DROP TABLE IF EXISTS {TABLE}") + + +def _partition_ids(backend): + return sorted( + row["partition_id"] + for row in backend.fetch_all( + f"SELECT partition_id FROM system.parts " + f"WHERE table = '{TABLE}' AND active" + ) + ) + + +def _row_count(backend): + return backend.fetch_all(f"SELECT count() AS n FROM {TABLE}")[0]["n"] + + +class TestClickHousePartitionKey: + """``PARTITION BY `` is the whole of ClickHouse's declarative surface.""" + + def test_partitions_appear_from_inserts_not_from_ddl(self, partitioned_backend): + assert _partition_ids(partitioned_backend) == ["202601", "202602"] + assert _row_count(partitioned_backend) == 3 + + def test_introspection_comes_from_system_parts(self, partitioned_backend): + """``supports_partition_metadata_introspection`` is honest about ``system.parts``.""" + assert partitioned_backend.dialect.supports_partition_metadata_introspection() is True + rows = partitioned_backend.fetch_all( + f"SELECT DISTINCT partition_id, partition FROM system.parts " + f"WHERE table = '{TABLE}' AND active ORDER BY partition_id" + ) + assert [r["partition_id"] for r in rows] == ["202601", "202602"] + assert [r["partition"] for r in rows] == ["202601", "202602"] + + +class TestPartitionMaintenanceClausesExecute: + """Each rendered clause is run against the server.""" + + def test_detach_then_attach_round_trips(self, partitioned_backend): + dialect = partitioned_backend.dialect + detach_sql, detach_params = ClickHouseDetachPartitionExpression( + dialect, TABLE, "202601" + ).to_sql() + assert detach_sql == f"ALTER TABLE `{TABLE}` DETACH PARTITION ID '202601'" + partitioned_backend.execute(detach_sql, detach_params) + assert _partition_ids(partitioned_backend) == ["202602"] + assert _row_count(partitioned_backend) == 1 + + attach_sql, attach_params = ClickHouseAttachPartitionExpression( + dialect, TABLE, "202601" + ).to_sql() + assert attach_sql == f"ALTER TABLE `{TABLE}` ATTACH PARTITION ID '202601'" + partitioned_backend.execute(attach_sql, attach_params) + assert _partition_ids(partitioned_backend) == ["202601", "202602"] + assert _row_count(partitioned_backend) == 3 + + def test_drop_removes_the_partition(self, partitioned_backend): + sql, params = ClickHouseDropPartitionExpression( + partitioned_backend.dialect, TABLE, "202601" + ).to_sql() + partitioned_backend.execute(sql, params) + assert _partition_ids(partitioned_backend) == ["202602"] + assert _row_count(partitioned_backend) == 1 + + +class TestMySQLPartitionStatementsAreRejectedByTheServer: + """The statements the backend used to stub, checked against the server itself. + + This is the ``VECTOR(4)`` argument made concrete: a formatter that renders + ``ALTER TABLE t TRUNCATE PARTITION p`` is not a fail-fast surface, it is a + promise the server cannot keep. The error text is what proves the absence — + ClickHouse answers with the list of clauses it *does* accept. + """ + + def _refused(self, backend, sql): + with pytest.raises(Exception) as excinfo: + backend.execute(sql) + return str(excinfo.value) + + def test_add_partition_is_not_a_clickhouse_clause(self, partitioned_backend): + message = self._refused( + partitioned_backend, + f"ALTER TABLE {TABLE} ADD PARTITION " + "(PARTITION p2026_03 VALUES LESS THAN ('2026-04-01'))", + ) + assert "Expected one of: COLUMN, INDEX, STATISTICS, PROJECTION, CONSTRAINT" in message + + def test_truncate_partition_is_not_a_clickhouse_clause(self, partitioned_backend): + message = self._refused(partitioned_backend, f"ALTER TABLE {TABLE} TRUNCATE PARTITION p") + # ClickHouse echoes the failing token back, so only the expectation list + # after "Expected one of:" is evidence. It enumerates every ALTER TABLE + # clause ClickHouse accepts, and none of them is TRUNCATE PARTITION. + expected = message.split("Expected one of:", 1)[1] + assert "DROP PARTITION" in expected + assert "DETACH PARTITION" in expected + assert "ATTACH PARTITION" in expected + assert "DELETE" in expected + assert "TRUNCATE" not in expected + + def test_subpartition_by_is_not_a_clickhouse_clause(self, partitioned_backend): + message = self._refused(partitioned_backend, f"ALTER TABLE {TABLE} SUBPARTITION BY HASH(id)") + assert "SUBPARTITION" in message + assert "Expected one of" in message + + def test_mysql_partition_ddl_is_not_a_clickhouse_clause(self, partitioned_backend): + message = self._refused( + partitioned_backend, + "CREATE TABLE ar_ch_mysql_part (id UInt32, created_at DateTime) " + "ENGINE = MergeTree " + "PARTITION BY RANGE COLUMNS (created_at) " + "(PARTITION p2026_01 VALUES LESS THAN ('2026-02-01')) " + "ORDER BY id", + ) + assert "Syntax error" in message + assert "COLUMNS" in message + + def test_information_schema_partitions_does_not_exist(self, partitioned_backend): + message = self._refused( + partitioned_backend, "SELECT * FROM information_schema.PARTITIONS LIMIT 1" + ) + assert "information_schema.PARTITIONS" in message + + def test_delete_in_partition_is_the_truncate_substitute(self, partitioned_backend): + """What a caller reaches for instead of ``TRUNCATE PARTITION``.""" + partitioned_backend.execute( + f"ALTER TABLE {TABLE} DELETE IN PARTITION ID '202601' WHERE 1" + ) + assert _row_count(partitioned_backend) == 1 + + +class TestGenericPartitionExpressionsRefuseWithTheClickHouseAlternative: + """The dialect-level refusal, with its suggestion naming where PARTITION BY is.""" + + def test_generic_partition_clause_refuses(self, dialect): + with pytest.raises(UnsupportedFeatureError) as excinfo: + PartitionClause( + dialect, PartitionStrategy.RANGE, [Column(dialect, "created_at")] + ).to_sql() + assert "PARTITION BY " in excinfo.value.suggestion + + def test_inline_partition_definition_refuses(self, dialect): + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.format_partition_definition(PartitionDefinition(name="p1")) + assert "PARTITION BY " in excinfo.value.suggestion \ No newline at end of file diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_column_type.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_column_type.py new file mode 100644 index 0000000..cdfa4a6 --- /dev/null +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_column_type.py @@ -0,0 +1,244 @@ +# tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_column_type.py +"""ClickHouse's eighteen-entry column-type table. + +Pure dialect tests -- no server. What is asserted here is the *declaration*: +that the dialect answers every entry of the protocol's common-type list, that +the cells are the ones the 26.7.3.19 probe measured, and that nothing is +refused without a written-down reason. The rendering those cells imply is +Phase 2b's work and is deliberately not asserted here. + +The answers are read through the model layer's own selection +(``resolve_column_class``) wherever that is possible, because a table that +answers correctly but is not reachable by the selection would fail only at +query time. +""" + +import datetime +import decimal +import enum +import uuid + +import pytest +from rhosocial.activerecord.backend.dialect.mixins import ColumnTypeMixin +from rhosocial.activerecord.backend.expression.column_types import ( + ArrayColumn, + BinaryColumn, + BooleanColumn, + ColumnBase, + IntegerColumn, + JSONColumn, + NumericColumn, + StringColumn, + TimestampColumn, + UUIDColumn, +) +from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect +from rhosocial.activerecord.backend.impl.clickhouse.mixins import ( + ClickHouseColumnTypeMixin, +) +from rhosocial.activerecord.testsuite.feature.query.typed_column.column_helpers import ( + COMMON_TYPES, + resolve_column_class, +) + +#: The version every probe behind this table ran on. +MEASURED = (26, 7, 3) + +#: ``{entry: answer}`` as ClickHouse states it. Written out in full rather than +#: derived, because a derived table would restate the code's own answer back at +#: itself and could not catch a change to it. +_CLICKHOUSE_ANSWERS = [ + (bool, BooleanColumn), + (int, IntegerColumn), + (float, NumericColumn), + (decimal.Decimal, NumericColumn), + (str, StringColumn), + (bytes, BinaryColumn), + (bytearray, BinaryColumn), + (datetime.date, TimestampColumn), + (datetime.time, TimestampColumn), + (datetime.datetime, TimestampColumn), + (datetime.timedelta, NumericColumn), + (uuid.UUID, UUIDColumn), + (dict, JSONColumn), + (list, ArrayColumn), + (tuple, ArrayColumn), + (set, ArrayColumn), + (frozenset, ArrayColumn), + (enum.Enum, StringColumn), +] + +_ENTRY_IDS = [getattr(entry, "__name__", str(entry)) for entry, _ in _CLICKHOUSE_ANSWERS] + +#: The entries this backend answers ``None`` for, each with the reason it was +#: measured to be inexpressible. Empty: the 26.7.3.19 probe found a working +#: pairing for all eighteen -- ClickHouse has a native array, a native JSON type +#: and ``ILIKE``, which are exactly the three things the backends that *do* +#: refuse lack. Kept as a declaration so that adding a ``None`` later means +#: adding its reason here, and so a ``None`` nobody justified is a test failure +#: rather than a quiet gap. +_REFUSALS: dict = {} + + +@pytest.fixture +def dialect(): + return ClickHouseDialect(version=MEASURED) + + +class TestTableCompleteness: + """A hole in the table is a silent gap, and the protocol forbids one.""" + + def test_the_dialect_carries_the_mixin(self, dialect): + assert isinstance(dialect, ClickHouseColumnTypeMixin) + assert isinstance(dialect, ColumnTypeMixin) + + def test_the_clickhouse_half_comes_first_in_the_mro(self, dialect): + """The load-bearing part of the dialect's base list. + + ``ClickHouseColumnTypeMixin`` overrides ``suggested_column_types()``, so + C3 has to see it before ``ColumnTypeMixin`` or the answer would be the + NotImplementedError core raises. Nothing in a query would look wrong + either way -- the failure names the table, not the composition -- which + is exactly why it takes an assertion to notice. + """ + mro = type(dialect).__mro__ + assert mro.index(ClickHouseColumnTypeMixin) < mro.index(ColumnTypeMixin) + + def test_every_entry_of_the_contract_is_answered(self, dialect): + """All eighteen, by name -- an omission fails here rather than at lookup.""" + table = dialect.suggested_column_types() + assert [e for e in COMMON_TYPES if e not in table] == [] + + def test_the_table_has_no_entry_beyond_the_contract(self, dialect): + """No extra key of ClickHouse's own; an extension would be a deliberate act.""" + table = dialect.suggested_column_types() + assert set(table) == set(COMMON_TYPES) + + def test_every_answer_is_a_class_or_none(self, dialect): + """The protocol's two answer states, and no third one.""" + table = dialect.suggested_column_types() + malformed = [ + (getattr(entry, "__name__", str(entry)), value) + for entry, value in table.items() + if value is not None and not (isinstance(value, type) and issubclass(value, ColumnBase)) + ] + assert malformed == [] + + def test_no_entry_is_refused_without_a_reason(self, dialect): + """``None`` is a last resort and must be justified, or answered as a class. + + The probe found a pairing for all eighteen here, so the honest answer is + a class for all of them: Firebird's ``dict`` and ``list`` (no JSON + functions, no arrays at all) and MySQL 5.6's (no JSON functions) have + something to refuse and ClickHouse does not. + """ + table = dialect.suggested_column_types() + refused = [entry for entry in COMMON_TYPES if table[entry] is None] + assert refused == [] + assert set(_REFUSALS) == {e for e in COMMON_TYPES if table[e] is None} + + @pytest.mark.parametrize("entry", COMMON_TYPES, ids=_ENTRY_IDS) + def test_every_answer_is_a_column_class(self, dialect, entry): + answer = dialect.suggested_column_types()[entry] + assert isinstance(answer, type) + assert issubclass(answer, ColumnBase) + + @pytest.mark.parametrize("entry", COMMON_TYPES, ids=_ENTRY_IDS) + def test_every_entry_resolves_through_the_selection(self, dialect, entry): + """Through the model layer's own lookup, not by reading the dict.""" + assert resolve_column_class(dialect, entry) is (dialect.suggested_column_types()[entry]) + + @pytest.mark.parametrize("entry", COMMON_TYPES, ids=_ENTRY_IDS) + def test_every_answer_builds_a_column(self, dialect, entry): + """A class in the table is only real if a column can be built from it.""" + column = resolve_column_class(dialect, entry)(dialect, "c") + assert isinstance(column, ColumnBase) + + def test_the_table_is_a_copy(self, dialect): + """Mutating the answer must not corrupt the table the next lookup reads.""" + first = dialect.suggested_column_types() + first[str] = IntegerColumn + assert dialect.suggested_column_types()[str] is StringColumn + + +class TestMeasuredCells: + """The 26.7.3.19 answers, cell by cell.""" + + @pytest.mark.parametrize("entry, expected", _CLICKHOUSE_ANSWERS, ids=_ENTRY_IDS) + def test_measured_entry(self, dialect, entry, expected): + assert dialect.suggested_column_types()[entry] is expected + + @pytest.mark.parametrize("entry, expected", _CLICKHOUSE_ANSWERS, ids=_ENTRY_IDS) + def test_measured_entry_through_the_selection(self, dialect, entry, expected): + assert resolve_column_class(dialect, entry) is expected + + def test_int_stays_integer_column(self, dialect): + """议题 C is open: this is not a ruling that ClickHouse has no UInt class. + + ClickHouse has ``UInt8``..``UInt256``. Whether "non-negative only" earns + a column class of its own is undecided, so ``int`` answers the whole + number every backend answers. The test exists so that a future ruling is + a deliberate change here rather than an accident. + """ + assert resolve_column_class(dialect, int) is IntegerColumn + + def test_the_numeric_family_answers_the_one_numeric_class(self, dialect): + """``float``, ``Decimal`` and ``timedelta`` all answer NumericColumn. + + The measurement behind the float and Decimal cells is that they are + genuinely different surfaces on this server: Float64 yields inf on + overflow and carries nan natively, and ``AVG`` of a Decimal answers + Float64 rather than a Decimal. That difference is real but it is not a + *column* difference any more -- core carries one class for every numeric + width and precision, and the difference between a ``Decimal(18, 4)`` and + a ``Float64`` belongs to the DDL layer's DataType, which this table never + reads. A duration is a number of seconds here, so it answers the same + class as the other numbers. + """ + assert resolve_column_class(dialect, float) is NumericColumn + assert resolve_column_class(dialect, decimal.Decimal) is NumericColumn + assert resolve_column_class(dialect, datetime.timedelta) is NumericColumn + + def test_time_is_provisional(self, dialect): + """Core has no TimeColumn yet, so the shared TimestampColumn stands. + + Worth pinning because it is wrong twice over in the meantime: this + backend's own ``TimeType`` still renders ``DateTime`` while ClickHouse + has had ``Time``/``Time64`` for a while (investigation appendix D-①). + """ + assert resolve_column_class(dialect, datetime.time) is TimestampColumn + + +class TestArrayCells: + """``Array(T)`` is native here, which is the whole reason for these cells.""" + + @pytest.mark.parametrize("entry", [list, tuple, set, frozenset]) + def test_container_entry_is_an_array(self, dialect, entry): + assert resolve_column_class(dialect, entry) is ArrayColumn + + def test_tuple_carries_no_narrowing(self, dialect): + """议题 A is open and this cell is its subject, not its answer. + + ClickHouse's ``Tuple`` is a heterogeneous positional group and core has + no column class for it. ``ArrayColumn`` is the agreed table's answer and + the permissive fallback this entry used to reach is gone; if 议题 A + decides ``Tuple`` gets a class, this moves with it. + """ + assert resolve_column_class(dialect, tuple) is ArrayColumn + + +class TestJSONCell: + """``dict`` is JSONColumn at every version, and no operation is gated.""" + + def test_dict_is_json_column(self, dialect): + assert resolve_column_class(dialect, dict) is JSONColumn + + def test_dict_is_json_column_on_an_old_version_too(self): + """Below the JSON type it is still answerable -- hence never refused. + + What carries it there is the ``JSONExtract`` / ``JSONHas`` / + ``isValidJSON`` function family over ``String`` storage, not the type. + Refusing it would be a limit ClickHouse does not have. + """ + old = ClickHouseDialect(version=(25, 8, 0)) + assert resolve_column_class(old, dict) is JSONColumn diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_dialect_security.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_dialect_security.py index 969539e..bdf594b 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_dialect_security.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_dialect_security.py @@ -144,21 +144,32 @@ def test_clickhouse_json_table_unsupported_validates_expression(dialect): def test_clickhouse_format_cast_expression_valid(dialect): - """Test that CAST expression validates target_type.""" - from rhosocial.activerecord.backend.expression.core import CastExpression, Literal - inner = Literal(dialect, "column") - expr = CastExpression(dialect, inner, "INTEGER") + """A CAST target is a DataType, and the dialect renders what it says. + + ClickHouse's own type rather than a CustomType: it implements none of the + generic format_data_type_* names, so a target it does not model is a target + it cannot render. That is the capability difference, not a gap in the test. + """ + from rhosocial.activerecord.backend.expression.core import CastExpression, Column + from rhosocial.activerecord.backend.impl.clickhouse.types import ClickHouseInt64Type + + expr = CastExpression(dialect, Column(dialect, "column"), ClickHouseInt64Type(dialect)) sql, params = dialect.format_cast_expression(expr) - assert "INTEGER" in sql + assert "Int64" in sql def test_clickhouse_format_cast_expression_rejects_injection(dialect): - """Test that malicious target_type is rejected.""" - from rhosocial.activerecord.backend.expression.core import CastExpression, Literal - inner = Literal(dialect, "column") - expr = CastExpression(dialect, inner, "INTEGER; DROP TABLE users--") - with pytest.raises(ValueError, match="Invalid target type"): - dialect.format_cast_expression(expr) + """A string in the type position is refused at construction. + + The type position cannot be bound as a parameter, so whatever lands there is + rendered into the statement text. That is why a string is not accepted at + all rather than being escaped: there is nothing to escape it into, since it + is code and not data.""" + from rhosocial.activerecord.backend.expression.core import CastExpression, Column + + with pytest.raises(TypeError, match="takes a DataType instance"): + CastExpression(dialect, Column(dialect, "column"), + "INTEGER; DROP TABLE users--") class TestClickHouseEscapeSqlStringBackslash: diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_mixins_coverage.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_mixins_coverage.py index 3ea1387..87f2ce2 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_mixins_coverage.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_mixins_coverage.py @@ -89,10 +89,97 @@ def test_table_partitioning_supported(self, dialect): def test_partitioned_table_creation_supported(self, dialect): assert dialect.supports_partitioned_table_creation() is True + def test_partition_metadata_introspection_supported(self, dialect): + """ClickHouse reports partitions in ``system.parts``, not information_schema.""" + assert dialect.supports_partition_metadata_introspection() is True + def test_mysql_partition_types_unsupported(self, dialect): + """The MySQL strategies are gone; the generic contract still answers False.""" assert dialect.supports_range_table_partitioning() is False assert dialect.supports_list_table_partitioning() is False assert dialect.supports_hash_table_partitioning() is False + assert dialect.supports_subpartitioning() is False + + def test_partition_maintenance_supported(self, dialect): + """``ALTER TABLE ... DROP/DETACH/ATTACH PARTITION ID`` are ClickHouse's own.""" + assert dialect.supports_drop_partition() is True + assert dialect.supports_detach_partition() is True + assert dialect.supports_attach_partition() is True + + def test_mysql_partition_statement_switches_are_gone(self, dialect): + """No ``supports_*`` switch exists for a statement ClickHouse lacks. + + One switch per MySQL statement would be an inventory of statements this + server has never had; the authoritative inventory is the ALTER ... PARTITION + reference. Absence of the method is the honest answer. + """ + for name in ( + "supports_range_columns_partitioning", + "supports_list_columns_partitioning", + "supports_key_table_partitioning", + "supports_linear_hash_partitioning", + "supports_linear_key_partitioning", + "supports_partition_definition_options", + "supports_partition_value_maxvalue", + "supports_add_partition", + "supports_truncate_partition", + "supports_reorganize_partition", + "supports_remove_partitioning", + "supports_coalesce_partition", + "supports_exchange_partition", + "supports_analyze_partition", + "supports_check_partition", + "supports_optimize_partition", + "supports_rebuild_partition", + "supports_repair_partition", + ): + assert not hasattr(dialect, name), name + + def test_mysql_partition_formatters_are_gone(self, dialect): + for name in ( + "format_partition_by_range", + "format_partition_by_range_columns", + "format_partition_by_list", + "format_partition_by_list_columns", + "format_partition_by_hash", + "format_partition_by_key", + "format_partition_definition_options", + "format_partition_value", + "format_partition_name_list", + "format_subpartition_by", + "format_subpartition_definition", + "format_add_partition_statement", + "format_truncate_partition_statement", + "format_reorganize_partition_statement", + "format_exchange_partition_statement", + "format_remove_partitioning_statement", + "format_coalesce_partition_statement", + "format_analyze_partition_statement", + "format_check_partition_statement", + "format_optimize_partition_statement", + "format_rebuild_partition_statement", + "format_repair_partition_statement", + "format_get_partitions_expression", + ): + assert not hasattr(dialect, name), name + + def test_inventory_records_what_clickhouse_does_have(self, dialect): + """The ALTER-partition inventory is kept as data, so it can be asserted on.""" + from rhosocial.activerecord.backend.impl.clickhouse.mixins.partition import ( + ALTER_PARTITION_INVENTORY, + MYSQL_ONLY_PARTITION_STATEMENTS, + MYSQL_ONLY_PARTITION_SYNTAX, + ) + + assert "DETACH PARTITION|PART" in ALTER_PARTITION_INVENTORY + assert "DROP PARTITION|PART" in ALTER_PARTITION_INVENTORY + assert "ATTACH PARTITION|PART" in ALTER_PARTITION_INVENTORY + # No MySQL partition statement appears in the ClickHouse inventory. + for statement in MYSQL_ONLY_PARTITION_STATEMENTS: + assert statement not in ALTER_PARTITION_INVENTORY, statement + # Nor does any MySQL inline boundary spelling. + for syntax in MYSQL_ONLY_PARTITION_SYNTAX: + assert syntax not in ALTER_PARTITION_INVENTORY, syntax class TestExplainTypesCoverage: diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_unsupported_features.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_unsupported_features.py index f8b294d..5c6c445 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_unsupported_features.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/dialect/test_unsupported_features.py @@ -3,12 +3,26 @@ Fast-fail contract tests for ClickHouse-unsupported feature stubs. ClickHouse does not support a large MySQL/SQL-standard feature surface -(triggers, spatial types, VECTOR, SET, stored routines, LOAD XML, +(triggers, spatial types, SET, stored routines, LOAD XML, admin commands, TABLE/VALUES constructors, whole-table maintenance, optimizer hints, JSON Duality Views, FULLTEXT, JSON_TABLE). The dialect mixins for these features are fail-fast stubs: ``supports_*`` returns ``False`` and ``format_*`` raises :class:`UnsupportedFeatureError`. +Declarative partitioning is the one surface that was **deleted** rather than +stubbed, because ClickHouse has something with the same name: it partitions a +MergeTree table with ``PARTITION BY `` and maintains the result by +partition id. The MySQL statements it used to stub are absent from the +ClickHouse ``ALTER TABLE ... PARTITION`` inventory rather than switched off +one at a time — see ``TestDeclarativePartitionStub`` below. + +MySQL's ``VECTOR`` family is deliberately **absent** rather than stubbed: +ClickHouse has a vector type of its own (``QBit``), which this backend +renders as ``ClickHouseVectorType``, so a ``supports_vector_type() -> False`` +would contradict the ``supports_data_type_clickhouse_vector() -> True`` next +to it. See ``TestMySQLVectorSurfaceIsGone`` in +``tests/.../backend/types/test_clickhouse_type_protocol.py``. + These tests verify that contract without a database connection by building the relevant expression node and rendering it via ``to_sql()``, so the real expression-to-formatter dispatch path is exercised. @@ -23,7 +37,13 @@ Table, Trigger, ) +from rhosocial.activerecord.backend.expression.core import Column from rhosocial.activerecord.backend.expression.statements import OnConflictClause +from rhosocial.activerecord.backend.expression.statements.ddl_partition import ( + PartitionClause, + PartitionDefinition, + PartitionStrategy, +) from rhosocial.activerecord.backend.expression.statements.ddl_trigger import ( CreateTriggerExpression, DropTriggerExpression, @@ -70,10 +90,6 @@ ClickHouseTableExpression, ClickHouseValuesExpression, ) -from rhosocial.activerecord.backend.impl.clickhouse.expression.vector import ( - ClickHouseCreateVectorIndexExpression, - ClickHouseVectorLiteralExpression, -) @pytest.fixture(scope="module") @@ -130,20 +146,6 @@ def test_format_create_spatial_index_raises(self, dialect): ClickHouseCreateSpatialIndexExpression(dialect, "idx", "tbl", "col").to_sql() -class TestVectorStub: - def test_supports_flags(self, dialect): - assert dialect.supports_vector_type() is False - assert dialect.supports_vector_index() is False - - def test_format_vector_literal_raises(self, dialect): - with pytest.raises(UnsupportedFeatureError): - ClickHouseVectorLiteralExpression(dialect, [1.0, 2.0]).to_sql() - - def test_format_create_vector_index_raises(self, dialect): - with pytest.raises(UnsupportedFeatureError): - ClickHouseCreateVectorIndexExpression(dialect, "idx", "tbl", "col").to_sql() - - class TestOptimizerHintStub: def test_supports_flags(self, dialect): assert dialect.supports_optimizer_hint() is False @@ -279,3 +281,29 @@ def test_supports_on_conflict(self, dialect): def test_format_on_conflict_raises(self, dialect): with pytest.raises(UnsupportedFeatureError): OnConflictClause(dialect, None, do_nothing=True).to_sql() + + +class TestDeclarativePartitionStub: + """MySQL declarative partitioning is gone from this dialect. + + ClickHouse partitions a ``MergeTree`` table with ``PARTITION BY `` in + ``CREATE TABLE`` — an arbitrary expression, not a chosen RANGE/LIST/HASH/KEY + strategy with ``VALUES`` boundaries — so a generic ``PartitionClause`` and + an inline ``PartitionDefinition`` both have nothing to render. What + ClickHouse *does* have is maintenance by partition id, which is covered by + ``ClickHousePartitionMixin`` and + ``tests/.../backend/expression/test_expression_signatures.py``. + """ + + def test_partition_clause_raises(self, dialect): + with pytest.raises(UnsupportedFeatureError) as excinfo: + PartitionClause( + dialect, PartitionStrategy.RANGE, [Column(dialect, "created_at")] + ).to_sql() + # The refusal points at where PARTITION BY is actually written. + assert "PARTITION BY " in excinfo.value.suggestion + + def test_inline_partition_definition_raises(self, dialect): + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.format_partition_definition(PartitionDefinition(name="p1")) + assert "PARTITION BY " in excinfo.value.suggestion diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_datetime_interval_expressions.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_datetime_interval_expressions.py index 42344cb..f5f1bf7 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_datetime_interval_expressions.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_datetime_interval_expressions.py @@ -14,6 +14,7 @@ interval, ) from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect +from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ClickHouseInt64Type class TestClickHouseDateTimeIntervalExpressions: @@ -127,13 +128,16 @@ def test_alias_and_cast(self, clickhouse_dialect: ClickHouseDialect): Column(clickhouse_dialect, "started_at"), Column(clickhouse_dialect, "ended_at"), ) - .cast("SIGNED") + .cast(ClickHouseInt64Type(clickhouse_dialect)) .as_("elapsed_days") ) sql, params = expr.to_sql() - assert sql == "CAST(dateDiff(%s, `started_at`, `ended_at`) AS SIGNED) AS `elapsed_days`" + # ClickHouse spells a 64-bit signed integer Int64. SIGNED is MySQL + # syntax, and asserting it here had been checking a word this dialect + # never emits rather than the type it was given. + assert sql == "CAST(dateDiff(%s, `started_at`, `ended_at`) AS Int64) AS `elapsed_days`" assert params == ("DAY",) def test_query_expression_integration(self, clickhouse_dialect: ClickHouseDialect): diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_expression_signatures.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_expression_signatures.py index 7d10871..832e18f 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_expression_signatures.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_expression_signatures.py @@ -5,53 +5,91 @@ This module verifies that expression classes follow the format signature compliance pattern where each expression declares a format_method property that returns the name of the dialect method used to render it. + +The partition expressions it previously covered +(``ClickHousePartitionNameListExpression``, a MySQL ``p0, p1, ...`` name list) +are gone: ClickHouse has no DDL-declared partition names to list. What is +covered now is the partition-id clause family, which is ClickHouse's own. """ import pytest from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect from rhosocial.activerecord.backend.impl.clickhouse.expression.partition import ( - ClickHousePartitionNameListExpression, + ClickHouseAttachPartitionExpression, + ClickHouseDetachPartitionExpression, + ClickHouseDropPartitionExpression, ) -class TestClickHousePartitionNameListExpression: - """Test ClickHousePartitionNameListExpression class.""" +@pytest.fixture(scope="module") +def dialect(): + """A dialect carrying the scenario server's version shape (26.7.3).""" + return ClickHouseDialect(version=(26, 7, 3)) + + +class TestClickHousePartitionIdExpressions: + """Signature compliance for the three ``ALTER TABLE ... PARTITION ID`` clauses.""" - def test_partition_name_list_basic(self): - """Test basic partition name list expression creation.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + EXPRESSIONS = ( + (ClickHouseDropPartitionExpression, "format_drop_partition_statement", "DROP"), + (ClickHouseDetachPartitionExpression, "format_detach_partition_statement", "DETACH"), + (ClickHouseAttachPartitionExpression, "format_attach_partition_statement", "ATTACH"), + ) - expr = ClickHousePartitionNameListExpression( - dialect, partitions=["2024_01", "2024_02", "2024_03"] - ) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_format_method_names_the_dialect_formatter(self, dialect, expr_class, formatter, verb): + """Each class declares exactly the ``format_*`` that renders it.""" + expr = expr_class(dialect, "visits", "202601") + assert expr.format_method == formatter + # Read off the class too, the way the structural sweep does. + assert expr_class.format_method.fget(expr_class) == formatter + # ...and the declared constant stays in step with the verb the shared + # renderer uses, so the three clauses cannot drift apart. + assert formatter == f"format_{expr_class.verb.lower()}_partition_statement" - assert expr.partitions == ["2024_01", "2024_02", "2024_03"] - assert expr.format_method == "format_partition_name_list" + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_declared_formatter_exists_on_the_dialect(self, dialect, expr_class, formatter, verb): + assert callable(getattr(dialect, formatter, None)) - def test_partition_name_list_empty_raises(self): - """Test that empty partitions list raises ValueError.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_renders_the_clickhouse_clause(self, dialect, expr_class, formatter, verb): + """Rendering targets the ``PARTITION ID`` spelling, quoted as a literal.""" + expr = expr_class(dialect, "visits", "202601") + sql, params = expr.to_sql() + assert sql == f"ALTER TABLE `visits` {verb} PARTITION ID '202601'" + assert params == () - with pytest.raises(ValueError, match="partitions must not be empty"): - ClickHousePartitionNameListExpression(dialect, partitions=[]) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_partition_id_is_stored_verbatim(self, dialect, expr_class, formatter, verb): + """The id is kept as given, so ``system.parts.partition_id`` is the source of truth.""" + expr = expr_class(dialect, "visits", "202601_3_3_0") + assert expr.partition_id == "202601_3_3_0" + assert "'202601_3_3_0'" in expr.to_sql()[0] - def test_partition_name_list_non_string_raises(self): - """Test that non-string partition names raise TypeError.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_quote_inside_partition_id_is_escaped(self, dialect, expr_class, formatter, verb): + """An embedded quote is doubled, not passed through (see format_literal).""" + sql, _ = expr_class(dialect, "visits", "a'b").to_sql() + assert "'a''b'" in sql - with pytest.raises(TypeError, match="partition name at index 1 must be a string"): - ClickHousePartitionNameListExpression( - dialect, partitions=["2024_01", 123, "2024_03"] - ) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_schema_qualifies_the_table(self, dialect, expr_class, formatter, verb): + """A ``schema`` argument renders the two-part ClickHouse identifier.""" + sql, _ = expr_class(dialect, "visits", "202601", schema="db1").to_sql() + assert sql.startswith("ALTER TABLE `db1`.`visits` ") - def test_partition_name_list_single_partition(self): - """Test partition name list with single partition.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_empty_table_raises(self, dialect, expr_class, formatter, verb): + with pytest.raises(ValueError, match="table must be a non-empty string"): + expr_class(dialect, " ", "202601") - expr = ClickHousePartitionNameListExpression( - dialect, partitions=["2024_01"] - ) + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_empty_partition_id_raises(self, dialect, expr_class, formatter, verb): + with pytest.raises(ValueError, match="partition_id must be a non-empty string"): + expr_class(dialect, "visits", " ") - assert expr.partitions == ["2024_01"] - assert len(expr.partitions) == 1 + @pytest.mark.parametrize("expr_class,formatter,verb", EXPRESSIONS) + def test_non_string_partition_id_raises(self, dialect, expr_class, formatter, verb): + with pytest.raises(TypeError, match="partition_id must be a string"): + expr_class(dialect, "visits", 202601) \ No newline at end of file diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_json_expressions.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_json_expressions.py index 07ffb6f..7fb31b9 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_json_expressions.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/expression/test_json_expressions.py @@ -23,7 +23,7 @@ class TestClickHouseJSONExtractExpression: def test_json_extract_basic(self): """Test basic JSON extraction.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONExtractExpression(dialect, "data", "$.name") sql, params = expr.to_sql() @@ -34,7 +34,7 @@ def test_json_extract_basic(self): def test_json_extract_with_alias(self): """Test JSON extraction with alias.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONExtractExpression(dialect, "data", "$.name").as_("extracted_name") sql, params = expr.to_sql() @@ -44,7 +44,7 @@ def test_json_extract_with_alias(self): def test_json_extract_array_path(self): """Test JSON extraction with array path.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONExtractExpression(dialect, "data", "$[0]") sql, params = expr.to_sql() @@ -58,7 +58,7 @@ class TestClickHouseJSONObjectExpression: def test_json_object_basic(self): """Test basic JSON object creation.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONObjectExpression(dialect, data={"name": "John", "age": 30}) sql, params = expr.to_sql() @@ -69,7 +69,7 @@ def test_json_object_basic(self): def test_json_object_with_kwargs(self): """Test JSON object creation with keyword arguments.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONObjectExpression(dialect, name="John", age=30).as_("obj") sql, params = expr.to_sql() @@ -83,7 +83,7 @@ class TestClickHouseJSONArrayExpression: def test_json_array_basic(self): """Test basic JSON array creation.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONArrayExpression(dialect, values=[1, 2, 3]) sql, params = expr.to_sql() @@ -93,7 +93,7 @@ def test_json_array_basic(self): def test_json_array_with_args(self): """Test JSON array with positional arguments.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONArrayExpression(dialect, 1, 2, 3) sql, params = expr.to_sql() @@ -103,7 +103,7 @@ def test_json_array_with_args(self): def test_json_array_with_alias(self): """Test JSON array with alias.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONArrayExpression(dialect, ["a", "b"]).as_("arr") sql, params = expr.to_sql() @@ -117,7 +117,7 @@ class TestClickHouseJSONContainsExpression: def test_json_contains_basic(self): """Test basic JSON contains check.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONContainsExpression(dialect, "data", "John", "$.name") sql, params = expr.to_sql() @@ -128,7 +128,7 @@ def test_json_contains_basic(self): def test_json_contains_with_alias(self): """Test JSON contains with alias.""" - dialect = ClickHouseDialect(version=(8, 0, 0)) + dialect = ClickHouseDialect(version=(26, 7, 1)) expr = ClickHouseJSONContainsExpression(dialect, "data", "value", "$.key").as_("contains") sql, params = expr.to_sql() diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/functions/test_dialect_function_support.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/functions/test_dialect_function_support.py index e8ec111..c9d82a9 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/functions/test_dialect_function_support.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/functions/test_dialect_function_support.py @@ -14,21 +14,21 @@ class TestClickHouseFunctionSupportBasic: def test_supports_functions_returns_dict(self): """Test that supports_functions returns a dictionary.""" - dialect = ClickHouseDialect((8, 0, 0)) + dialect = ClickHouseDialect((26, 7, 1)) result = dialect.supports_functions() assert isinstance(result, dict) assert len(result) > 0 def test_supports_functions_all_values_are_bool(self): """Test that all values in the returned dict are booleans.""" - dialect = ClickHouseDialect((8, 0, 0)) + dialect = ClickHouseDialect((26, 7, 1)) result = dialect.supports_functions() for func_name, supported in result.items(): assert isinstance(supported, bool), f"Value for {func_name} is not bool" def test_core_functions_always_supported(self): """Test that core functions are marked as supported.""" - dialect = ClickHouseDialect((8, 0, 0)) + dialect = ClickHouseDialect((26, 7, 1)) result = dialect.supports_functions() core_functions = ["count", "sum_", "avg", "min_", "max_", "coalesce", "nullif"] for func in core_functions: @@ -37,7 +37,7 @@ def test_core_functions_always_supported(self): def test_sqlxml_constructors_are_not_plain_functions(self): """Test that standard SQL/XML constructors are not plain functions.""" - dialect = ClickHouseDialect((8, 0, 0)) + dialect = ClickHouseDialect((26, 7, 1)) result = dialect.supports_functions() sqlxml_constructors = [ "xmlparse", @@ -136,7 +136,7 @@ class TestClickHouseFunctionSupportPrivateMethod: def test_unknown_function_returns_true(self): """Test that unknown functions return True (no restriction).""" - dialect = ClickHouseDialect((8, 0, 0)) + dialect = ClickHouseDialect((26, 7, 1)) result = dialect._is_clickhouse_function_supported("unknown_function_xyz") assert result is True @@ -152,7 +152,7 @@ class TestClickHouseFunctionSupportIntegration: def test_function_dict_contains_core_functions(self): """Test that the result contains core functions (not MySQL wrappers).""" - dialect = ClickHouseDialect((8, 0, 0)) + dialect = ClickHouseDialect((26, 7, 1)) result = dialect.supports_functions() assert any(func in result for func in ["count", "sum_", "avg"]) @@ -161,9 +161,15 @@ def test_function_dict_contains_core_functions(self): assert "find_in_set" not in result def test_function_support_stable_across_versions(self): - """Core function support does not change across ClickHouse versions.""" - old_dialect = ClickHouseDialect(version=(5, 6, 0)) - new_dialect = ClickHouseDialect(version=(26, 0, 0)) + """Core function support does not change across ClickHouse versions. + + The two versions are real release lines — the oldest maintained LTS + (25.8) and the current stable line (26.7). They were ``(5, 6, 0)`` and + ``(26, 0, 0)``; ``5.6`` is a MySQL release number and no ClickHouse + server has reported it. + """ + old_dialect = ClickHouseDialect(version=(25, 8, 0)) + new_dialect = ClickHouseDialect(version=(26, 7, 1)) old_result = old_dialect.supports_functions() new_result = new_dialect.supports_functions() diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_show_output_shape.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_show_output_shape.py new file mode 100644 index 0000000..1915077 --- /dev/null +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_show_output_shape.py @@ -0,0 +1,596 @@ +# tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_show_output_shape.py +""" +The SHOW sub-introspector reads ClickHouse's output columns, not MySQL's. + +ClickHouse's ``SHOW`` commands report different column names from MySQL's, and +the difference is not cosmetic: reading MySQL's names against ClickHouse's rows +does not raise, it silently yields nothing. + +================================ ===================================== +statement ClickHouse columns +================================ ===================================== +``SHOW CREATE TABLE`` ``statement`` +``SHOW CREATE VIEW`` ``statement`` +``SHOW TABLES`` ``name`` +``SHOW FULL TABLES`` ``name``, ``engine`` +``SHOW DATABASES`` ``name`` +``SHOW PROCESSLIST`` ``system.processes``' 43 columns +``SHOW [FULL] COLUMNS`` ``field``, ``type``, ``null``, ``key``, + ``default``, ``extra`` (+ ``collation``, + ``comment``, ``privileges`` under FULL) +``SHOW INDEX`` ``table``, ``non_unique``, ``key_name``, + ``seq_in_index``, ``pk_col``, ``collation``, + ``cardinality``, ``sub_part``, ``packed``, + ``null``, ``index_type``, ``comment``, + ``index_comment``, ``visible``, + ``expression`` +``SHOW ENGINES`` ``name`` and the eight ``supports_*`` flags; + ``description``, ``syntax``, ``examples``, + ``introduced_in``, ``related`` only on + versions that publish them +``SHOW GRANTS`` one column named after the statement text + plus the requested output format +================================ ===================================== + +MySQL would answer ``Table``/``Create Table``, ``View``/``Create View``, +``Tables_in_``/``Table_type`` and ``Database``. Every row below is the +shape the server actually produces; the live half of this file +(``TestShowOutputAgainstTheServer``) re-reads it from a real connection. + +The three failure modes this pins down, all of which were live before: +``create_table``/``create_view`` returned ``create_statement=''``, +``tables(full=True)`` returned ``[]``, ``databases()`` returned one entry +per database with ``name=None``, and ``columns()`` / ``indexes()`` / +``processlist()`` / ``engines()`` / ``grants()`` raised +``UnsupportedFeatureError`` on statements the server accepts. + +MySQL would answer ``Table``/``Create Table``, ``View``/``Create View``, +``Tables_in_``/``Table_type``, ``Database``, ``Field``/``Type``/``Null``/ +``Key``, ``Column_name`` for the indexed column, ``Id``/``User``/``Host``/ +``Command``/``Time``, ``Engine``/``Support``/``Transactions``/``XA``/ +``Savepoints``, and ``Grants for`` — so the ones below are the ClickHouse +column names, not a restatement of the MySQL ones. +""" +import pytest + +from rhosocial.activerecord.backend.impl.clickhouse.introspection.show_introspector import ShowMixin + + +class TestCreateStatementOutputShape: + """``SHOW CREATE TABLE`` / ``SHOW CREATE VIEW`` answer a single ``statement`` column.""" + + TABLE_ROW = {"statement": "CREATE TABLE default.t\n(\n `id` Int64\n)\nENGINE = MergeTree"} + VIEW_ROW = {"statement": "CREATE VIEW default.v\n(\n `id` Int64\n)\nAS SELECT id"} + + def test_create_table_reads_the_statement_column(self): + result = ShowMixin._parse_create_table([self.TABLE_ROW], "t") + assert result.table_name == "t" + assert result.create_statement == self.TABLE_ROW["statement"] + + def test_create_view_reads_the_statement_column(self): + result = ShowMixin._parse_create_view([self.VIEW_ROW], "v") + assert result.view_name == "v" + assert result.create_statement == self.VIEW_ROW["statement"] + + def test_create_view_leaves_mysql_only_fields_empty(self): + result = ShowMixin._parse_create_view([self.VIEW_ROW], "v") + assert result.character_set_client is None + assert result.collation_connection is None + + def test_no_rows_returns_none(self): + assert ShowMixin._parse_create_table([], "t") is None + assert ShowMixin._parse_create_view([], "v") is None + + def test_mysql_column_names_are_not_consulted(self): + """A row carrying only MySQL's names must not be mistaken for a result.""" + assert ShowMixin._parse_create_table([{"Create Table": "..."}], "t").create_statement == "" + assert ShowMixin._parse_create_view([{"Create View": "..."}], "v").create_statement == "" + + +class TestListOutputShape: + """``SHOW TABLES`` / ``SHOW DATABASES`` name the column ``name``.""" + + def test_tables_reads_name(self): + rows = [{"name": "a"}, {"name": "b"}] + result = ShowMixin._parse_tables(rows) + assert [(r.name, r.table_type) for r in result] == [("a", None), ("b", None)] + + def test_full_tables_reads_name_and_engine(self): + """Under ``FULL`` the second column is the storage engine, not a table type.""" + rows = [{"name": "a", "engine": "MergeTree"}, {"name": "v", "engine": "View"}] + result = ShowMixin._parse_tables(rows) + assert [(r.name, r.table_type) for r in result] == [ + ("a", "MergeTree"), + ("v", "View"), + ] + + def test_databases_reads_name(self): + rows = [{"name": "default"}, {"name": "system"}] + assert [r.name for r in ShowMixin._parse_databases(rows)] == ["default", "system"] + + def test_mysql_column_names_are_not_consulted(self): + """MySQL's ``Tables_in_`` / ``Database`` names must not be looked for.""" + mysql_rows = [{"Tables_in_db": "a", "Table_type": "BASE TABLE"}] + result = ShowMixin._parse_tables(mysql_rows) + assert [(r.name, r.table_type) for r in result] == [(None, None)] + assert [r.name for r in ShowMixin._parse_databases([{"Database": "default"}])] == [None] + + +# Every row below is a row the 26.7.3.19 server returned, quoted column for +# column; the live classes at the bottom of this file re-derive them. + +COLUMN_ROWS = [ + {"field": "al", "type": "UInt8", "null": "NO", "key": "", "default": "id", "extra": ""}, + {"field": "id", "type": "UInt32", "null": "NO", "key": "PRI SOR", "default": None, "extra": ""}, +] + +COLUMN_FULL_ROWS = [ + { + "field": "id", "type": "UInt32", "null": "NO", "key": "PRI SOR", + "default": None, "extra": "", "collation": None, "comment": "", + "privileges": "", + } +] + +INDEX_ROWS = [ + { + "table": "idx_probe", "non_unique": 1, "key_name": "idx_s", + "seq_in_index": 1, "pk_col": "", "collation": None, "cardinality": 0, + "sub_part": None, "packed": None, "null": None, + "index_type": "BLOOM_FILTER", "comment": "", "index_comment": "", + "visible": "YES", "expression": "s", + }, + { + "table": "idx_probe", "non_unique": 1, "key_name": "PRIMARY", + "seq_in_index": 1, "pk_col": "id", "collation": "A", "cardinality": 0, + "sub_part": None, "packed": None, "null": None, + "index_type": "PRIMARY", "comment": "", "index_comment": "", + "visible": "YES", "expression": "", + }, +] + +PROCESSLIST_ROWS = [ + { + "is_initial_query": 1, "user": "root", + "query_id": "0a944dd1-8a7e-4fd2-9733-e9e44f4d9f53", + "address": "::ffff:192.168.65.1", "port": 39550, + "elapsed": 0.00164, "read_rows": 0, "read_bytes": 0, + "memory_usage": 4438556, "query": "SELECT 1", "query_kind": "Select", + "thread_ids": [42], "current_database": "test_db", "is_internal": 0, + } +] + +ENGINE_ROWS = [ + { + "name": "AggregatingMergeTree", "supports_settings": 1, + "supports_skipping_indices": 1, "supports_projections": 1, + "supports_sort_order": 1, "supports_ttl": 1, "supports_replication": 0, + "supports_deduplication": 0, "supports_parallel_insert": 1, + "description": "# AggregatingMergeTree", "syntax": "AGGREGATING", + "examples": "", "introduced_in": "", + "related": ["ReplicatedAggregatingMergeTree"], + } +] + +GRANT_ROWS = [ + {"GRANTS FORMAT Native": "GRANT SOURCES ON *.* TO root WITH GRANT OPTION"}, + {"GRANTS FOR root": "GRANT SET DEFINER ON * TO root WITH GRANT OPTION"}, +] + + +class TestColumnsOutputShape: + """``SHOW [FULL] COLUMNS`` answers lower-cased ``field``/``type``/``key``/...""" + + def test_columns_read_the_lowercase_names(self): + result = ShowMixin._parse_columns(COLUMN_ROWS) + assert [(r.field, r.type, r.null, r.key) for r in result] == [ + ("al", "UInt8", "NO", ""), + ("id", "UInt32", "NO", "PRI SOR"), + ] + + def test_columns_carry_default_and_extra(self): + result = ShowMixin._parse_columns(COLUMN_ROWS) + assert [(r.default, r.extra) for r in result] == [("id", ""), (None, "")] + + def test_full_columns_read_collation_comment_and_privileges(self): + """The three extra columns appear only under ``FULL``, in lower case.""" + result = ShowMixin._parse_columns(COLUMN_FULL_ROWS) + assert (result[0].collation, result[0].comment, result[0].privileges) == ( + None, "", "", + ) + + def test_plain_columns_leave_the_full_only_columns_empty(self): + result = ShowMixin._parse_columns(COLUMN_ROWS) + assert all(r.collation is None and r.comment is None and r.privileges is None for r in result) + + def test_mysql_column_names_are_not_consulted(self): + """MySQL's ``Field``/``Type``/``Key`` must not be looked for.""" + result = ShowMixin._parse_columns( + [{"Field": "id", "Type": "int", "Null": "NO", "Key": "PRI", "Default": None, "Extra": ""}] + ) + assert [(r.field, r.type, r.key) for r in result] == [(None, None, None)] + + +class TestIndexOutputShape: + """``SHOW INDEX`` answers ``pk_col`` where MySQL answers ``Column_name``.""" + + def test_index_reads_pk_col_as_the_column_name(self): + result = ShowMixin._parse_indexes(INDEX_ROWS) + assert [r.column_name for r in result] == ["", "id"] + + def test_index_reports_clickhouse_index_types(self): + result = ShowMixin._parse_indexes(INDEX_ROWS) + assert [r.index_type for r in result] == ["BLOOM_FILTER", "PRIMARY"] + + def test_skipping_index_keeps_its_expression(self): + skip, primary = ShowMixin._parse_indexes(INDEX_ROWS) + assert (skip.key_name, skip.expression, skip.collation) == ("idx_s", "s", None) + assert (primary.key_name, primary.expression, primary.collation) == ("PRIMARY", "", "A") + + def test_index_carries_no_mysql_btree_default(self): + """``index_type`` must come from the server, not default to ``BTREE``.""" + result = ShowMixin._parse_indexes([{"table": "t", "non_unique": 1, "key_name": "PRIMARY", "seq_in_index": 1}]) + assert result[0].index_type is None + + def test_mysql_column_names_are_not_consulted(self): + result = ShowMixin._parse_indexes( + [{"Table": "t", "Key_name": "PRIMARY", "Column_name": "id", "Index_type": "BTREE"}] + ) + assert [(r.table_name, r.column_name, r.index_type) for r in result] == [(None, None, None)] + + +class TestProcessListOutputShape: + """``SHOW PROCESSLIST`` is ``system.processes``, so its columns are that table's.""" + + def test_processlist_maps_query_id_user_address_and_elapsed(self): + row = ShowMixin._parse_processlist(PROCESSLIST_ROWS)[0] + assert row.id == "0a944dd1-8a7e-4fd2-9733-e9e44f4d9f53" + assert row.user == "root" + assert row.host == "::ffff:192.168.65.1:39550" + assert row.db == "test_db" + assert row.info == "SELECT 1" + assert row.time == pytest.approx(0.00164) + + def test_command_and_state_have_no_clickhouse_source(self): + """``system.processes`` has no per-process command or state column.""" + row = ShowMixin._parse_processlist(PROCESSLIST_ROWS)[0] + assert row.command is None + assert row.state is None + + def test_mysql_column_names_are_not_consulted(self): + result = ShowMixin._parse_processlist( + [{"Id": 1, "User": "root", "Host": "localhost", "Command": "Query", "Time": 0}] + ) + assert [(r.id, r.user, r.host, r.command, r.time) for r in result] == [ + (None, None, None, None, None) + ] + + +class TestEnginesOutputShape: + """``SHOW ENGINES`` is ``system.table_engines``: ``name`` plus ``supports_*``.""" + + def test_engines_read_the_name_column(self): + assert ShowMixin._parse_engines(ENGINE_ROWS)[0].engine == "AggregatingMergeTree" + + def test_engines_read_all_eight_supports_flags(self): + row = ShowMixin._parse_engines(ENGINE_ROWS)[0] + assert (row.supports_settings, row.supports_skipping_indices) == (1, 1) + assert (row.supports_projections, row.supports_sort_order) == (1, 1) + assert (row.supports_ttl, row.supports_replication) == (1, 0) + assert (row.supports_deduplication, row.supports_parallel_insert) == (0, 1) + + def test_engines_read_the_documentation_columns(self): + row = ShowMixin._parse_engines(ENGINE_ROWS)[0] + assert row.description == "# AggregatingMergeTree" + assert row.syntax == "AGGREGATING" + assert row.related == ["ReplicatedAggregatingMergeTree"] + + def test_mysql_column_names_are_not_consulted(self): + """MySQL's ``Engine``/``Support``/``XA`` must not be looked for.""" + result = ShowMixin._parse_engines( + [{"Engine": "InnoDB", "Support": "DEFAULT", "Transactions": "YES", "XA": "NO", "Savepoints": "NO"}] + ) + assert result[0].engine is None + + +class TestGrantsOutputShape: + """``SHOW GRANTS`` labels its only column after the statement plus the format.""" + + def test_grants_reads_the_native_protocol_column_name(self): + assert ShowMixin._parse_grants(GRANT_ROWS[:1])[0].grants.startswith("GRANT SOURCES") + + def test_grants_reads_the_for_user_column_name(self): + assert ShowMixin._parse_grants(GRANT_ROWS[1:])[0].grants == ( + "GRANT SET DEFINER ON * TO root WITH GRANT OPTION" + ) + + def test_grants_falls_back_to_the_single_column_value(self): + """HTTP with ``default_format=JSONEachRow`` names the bare column ``GRANTS``.""" + assert ShowMixin._parse_grants([{"GRANTS": "GRANT SELECT ON *.* TO u"}])[0].grants == ( + "GRANT SELECT ON *.* TO u" + ) + + def test_grants_reads_a_row_whose_column_is_unrecognised(self): + """The statement has exactly one column; its value is the grant either way.""" + assert ShowMixin._parse_grants([{"SHOW GRANTS": "GRANT ALL ON *.* TO u"}])[0].grants == ( + "GRANT ALL ON *.* TO u" + ) + + def test_mysql_column_name_is_not_required(self): + """MySQL's fixed ``Grants for`` is not the lookup key; the value is.""" + result = ShowMixin._parse_grants([{"Grants for root@localhost": "GRANT ALL"}]) + assert result[0].grants == "GRANT ALL" + + def test_an_empty_row_yields_no_grant(self): + assert ShowMixin._parse_grants([{}])[0].grants is None + + +class TestShowDialectRendersWhatTheServerAccepts: + """The five formatters emit SQL 26.7.3.19 parses, not MySQL's shapes.""" + + @pytest.fixture + def dialect(self): + from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect + + return ClickHouseDialect(version=(26, 7, 1)) + + def _columns(self, dialect, **kw): + from rhosocial.activerecord.backend.impl.clickhouse.expression.show import ( + ShowColumnsExpression, + ) + + expr = ShowColumnsExpression(dialect, "t") + if kw.get("schema"): + expr.schema(kw["schema"]) + if kw.get("full"): + expr.full() + if kw.get("like"): + expr.like(kw["like"]) + return expr.to_sql() + + def test_columns_sql(self, dialect): + assert self._columns(dialect) == ("SHOW COLUMNS FROM `t`", ()) + assert self._columns(dialect, schema="db") == ("SHOW COLUMNS FROM `db`.`t`", ()) + assert self._columns(dialect, full=True) == ("SHOW FULL COLUMNS FROM `t`", ()) + assert self._columns(dialect, like="i%") == ("SHOW COLUMNS FROM `t` LIKE %s", ("i%",)) + + def test_index_sql_uses_one_table_reference(self, dialect): + """``SHOW INDEX FROM t FROM db`` is a SYNTAX_ERROR; ``db.t`` is the form.""" + from rhosocial.activerecord.backend.impl.clickhouse.expression.show import ( + ShowIndexExpression, + ) + + expr = ShowIndexExpression(dialect, "t").schema("db") + assert expr.to_sql() == ("SHOW INDEX FROM `db`.`t`", ()) + + def test_processlist_sql_never_emits_full(self, dialect): + """``SHOW FULL PROCESSLIST`` is a SYNTAX_ERROR on 26.7.3.19.""" + from rhosocial.activerecord.backend.impl.clickhouse.expression.show import ( + ShowProcessListExpression, + ) + + assert ShowProcessListExpression(dialect).full().to_sql() == ("SHOW PROCESSLIST", ()) + + def test_engines_sql(self, dialect): + from rhosocial.activerecord.backend.impl.clickhouse.expression.show import ( + ShowEnginesExpression, + ) + + assert ShowEnginesExpression(dialect).to_sql() == ("SHOW ENGINES", ()) + + def test_grants_sql_drops_the_mysql_host_part(self, dialect): + """``SHOW GRANTS FOR root@localhost`` is UNKNOWN_ROLE on 26.7.3.19.""" + from rhosocial.activerecord.backend.impl.clickhouse.expression.show import ( + ShowGrantsExpression, + ) + + assert ShowGrantsExpression(dialect).to_sql() == ("SHOW GRANTS", ()) + assert ShowGrantsExpression(dialect).for_user("root").to_sql() == ( + "SHOW GRANTS FOR `root`", (), + ) + assert ShowGrantsExpression(dialect).for_user("root", "localhost").to_sql() == ( + "SHOW GRANTS FOR `root`", (), + ) + + +class TestShowOutputAgainstTheServer: + """The same shapes, read from a live ClickHouse connection.""" + + @pytest.fixture + def show_fixture(self, clickhouse_backend_single): + backend = clickhouse_backend_single + backend.execute("DROP TABLE IF EXISTS ar_show_shape") + backend.execute("DROP VIEW IF EXISTS ar_show_shape_v") + backend.execute( + "CREATE TABLE ar_show_shape (id UInt32, created_at DateTime) " + "ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY id" + ) + backend.execute( + "CREATE VIEW ar_show_shape_v AS SELECT id FROM ar_show_shape" + ) + yield backend + backend.execute("DROP VIEW IF EXISTS ar_show_shape_v") + backend.execute("DROP TABLE IF EXISTS ar_show_shape") + + @pytest.fixture + def indexed_fixture(self, clickhouse_backend_single): + """A table with a sorting key *and* a data skipping index.""" + backend = clickhouse_backend_single + backend.execute("DROP TABLE IF EXISTS ar_show_indexed") + backend.execute( + "CREATE TABLE ar_show_indexed (id UInt32, s String, " + "INDEX idx_s s TYPE bloom_filter(0.01) GRANULARITY 4) " + "ENGINE = MergeTree ORDER BY (id, s)" + ) + yield backend, "ar_show_indexed" + backend.execute("DROP TABLE IF EXISTS ar_show_indexed") + + def test_create_table_returns_the_statement(self, show_fixture): + result = show_fixture.introspector.show.create_table("ar_show_shape") + assert result.table_name == "ar_show_shape" + assert result.create_statement.startswith("CREATE TABLE") + assert "MergeTree" in result.create_statement + assert "PARTITION BY toYYYYMM(created_at)" in result.create_statement + + def test_create_view_returns_the_statement(self, show_fixture): + result = show_fixture.introspector.show.create_view("ar_show_shape_v") + assert result.view_name == "ar_show_shape_v" + assert result.create_statement.startswith("CREATE VIEW") + + def test_full_tables_reports_the_storage_engine(self, show_fixture): + rows = { + row.name: row.table_type + for row in show_fixture.introspector.show.tables(full=True) + } + assert rows["ar_show_shape"] == "MergeTree" + assert rows["ar_show_shape_v"] == "View" + + def test_tables_reports_names(self, show_fixture): + names = [row.name for row in show_fixture.introspector.show.tables()] + assert "ar_show_shape" in names + + def test_databases_reports_names(self, show_fixture): + names = [row.name for row in show_fixture.introspector.show.databases()] + assert "default" in names + assert all(name is not None for name in names) + + def test_columns_reports_types_and_sorting_key(self, show_fixture): + cols = {c.field: c for c in show_fixture.introspector.show.columns("ar_show_shape")} + assert set(cols) == {"id", "created_at"} + assert cols["id"].type == "UInt32" + assert cols["id"].null == "NO" + assert cols["id"].key == "PRI SOR" + assert cols["created_at"].type == "DateTime" + + def test_columns_respects_a_schema(self, show_fixture): + """``SHOW COLUMNS FROM db.t`` names the same columns as the bare form.""" + cols = show_fixture.introspector.show.columns( + "ar_show_shape", schema=show_fixture.config.database + ) + assert sorted(c.field for c in cols) == ["created_at", "id"] + + def test_columns_are_returned_in_name_order(self, show_fixture): + """ClickHouse orders the result by column name, not by declaration order. + + Measured: ``SHOW COLUMNS FROM t`` on a table declared + ``(id UInt32, created_at DateTime)`` returns ``created_at`` then ``id`` — + the definition order is only in ``SHOW CREATE TABLE`` / ``system.columns``. + """ + cols = show_fixture.introspector.show.columns("ar_show_shape") + assert [c.field for c in cols] == ["created_at", "id"] + + def test_full_columns_adds_comment_and_privileges(self, show_fixture): + cols = {c.field: c for c in show_fixture.introspector.show.columns("ar_show_shape", full=True)} + assert cols["id"].comment == "" + assert cols["id"].privileges == "" + # The server documents this column as always NULL: no per-column collations. + assert cols["id"].collation is None + + def test_columns_like_filters(self, show_fixture): + cols = show_fixture.introspector.show.columns("ar_show_shape", like="id%") + assert [c.field for c in cols] == ["id"] + + def test_columns_for_a_missing_table_is_empty_not_an_error(self, show_fixture): + """``SHOW COLUMNS FROM no_such_table`` returns headers and zero rows.""" + assert show_fixture.introspector.show.columns("ar_no_such_table_xyz") == [] + + def test_indexes_reports_sorting_key_and_skipping_index(self, indexed_fixture): + backend, table = indexed_fixture + rows = { + (r.key_name, r.seq_in_index): r + for r in backend.introspector.show.indexes(table) + } + primary_1 = rows[("PRIMARY", 1)] + primary_2 = rows[("PRIMARY", 2)] + assert (primary_1.column_name, primary_1.collation) == ("id", "A") + assert (primary_2.column_name, primary_2.collation) == ("s", "A") + skip = rows[("idx_s", 1)] + assert skip.index_type == "BLOOM_FILTER" + assert skip.column_name == "" + assert skip.expression == "s" + + def test_indexes_never_reports_btree(self, indexed_fixture): + backend, table = indexed_fixture + types = {r.index_type for r in backend.introspector.show.indexes(table)} + assert "BTREE" not in types + assert types <= {"PRIMARY", "BLOOM_FILTER"} + + def test_processlist_returns_rows_with_the_query_text(self, show_fixture): + rows = show_fixture.introspector.show.processlist() + assert isinstance(rows, list) + # Our own connection is excluded from system.processes, but the statement + # parses and the shape holds for whatever is running. + for row in rows: + assert row.id is not None + assert row.user is not None + assert row.command is None + assert row.state is None + + def test_engines_lists_the_table_engines(self, show_fixture): + """The eight ``supports_*`` flags, which every version publishes. + + Measured identical on 25.8.33.6, 26.3.28.5 and 26.7.3.19: + ``MergeTree`` is 1/1/1/1/1/0/0/1 and ``Memory`` is 1/0/0/0/0/0/0/1. + """ + engines = show_fixture.introspector.show.engines() + by_name = {e.engine: e for e in engines} + assert "MergeTree" in by_name + assert "Memory" in by_name + merge_tree = by_name["MergeTree"] + assert merge_tree.supports_skipping_indices == 1 + assert merge_tree.supports_ttl == 1 + assert merge_tree.supports_replication == 0 + + def test_engine_documentation_columns_follow_the_servers_schema( + self, show_fixture, clickhouse_backend_single + ): + """The documentation columns exist only where the server has them. + + Measured: ``system.table_engines`` carries those five columns on + 26.7.3.19 and 26.7.21.2, and carries **only** ``name`` plus the eight + ``supports_*`` flags on 25.8.33.6 and 26.3.28.5 — on 25.8 a literal + ``SELECT syntax FROM system.table_engines`` is rejected outright with + ``UNKNOWN_IDENTIFIER``. The exact release that added them was not + determined; only the 25.8/26.3 versus 26.7 boundary was. So the correct + behaviour differs by version, and the introspector must return ``None`` + for a column the server does not send rather than a synthesised string. + The assertion is made against the server's own column list, which is the + only version-independent statement available. + """ + columns = { + row["name"] + for row in show_fixture.execute( + "SELECT name FROM system.columns " + "WHERE database = 'system' AND table = 'table_engines'" + ).data + } + engines = show_fixture.introspector.show.engines() + merge_tree = next(e for e in engines if e.engine == "MergeTree") + for column, value in ( + ("syntax", merge_tree.syntax), + ("description", merge_tree.description), + ("related", merge_tree.related), + ): + if column in columns: + assert value, f"{column} is a column of system.table_engines but arrived empty" + else: + assert value is None, ( + f"{column} is not a column of system.table_engines on this " + f"version, so nothing may be reported for it: {value!r}" + ) + assert {"name", "supports_settings", "supports_replication"} <= columns + + def test_grants_returns_grant_statements(self, show_fixture): + grants = show_fixture.introspector.show.grants() + assert grants + assert all(g.grants and g.grants.startswith("GRANT ") for g in grants) + + def test_grants_for_a_named_role(self, show_fixture): + grants = show_fixture.introspector.show.grants(user="root", host="localhost") + assert grants + assert all(g.grants.startswith("GRANT ") for g in grants) + + def test_grants_for_a_missing_role_raises(self, show_fixture): + """The server, not this dialect, decides the role exists: UNKNOWN_ROLE.""" + with pytest.raises(Exception, match="ar_no_such_role_xyz"): + show_fixture.introspector.show.grants(user="ar_no_such_role_xyz") \ No newline at end of file diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_status_introspector.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_status_introspector.py new file mode 100644 index 0000000..8121a7f --- /dev/null +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_status_introspector.py @@ -0,0 +1,703 @@ +# tests/rhosocial/activerecord_clickhouse_test/feature/backend/introspection/test_status_introspector.py +""" +The status inventories report the server, they do not recite a list. + +Before this file the two inventories were MySQL's: of 45 ``CLICKHOUSE_CONFIG_ +VARIABLES`` exactly one (``connect_timeout``) was a ``system.settings`` row, and +of 31 ``CLICKHOUSE_STATUS_VARIABLES`` none was a ``system.metrics`` row. The +consequence was not an exception — it was ``list_performance_metrics()`` +returning ``[]`` on every call, forever. + +So the live half of this file is the point. It checks the inventories against +the table each name is supposed to come from, in **both** directions: + +* nothing is returned that the server does not have — the failure mode that + made the MySQL inventories return nothing and, once they had been padded + with placeholder rows, would have made them return fiction; +* nothing the server *does* have is dropped, so the inventories are not quietly + emptying themselves. + +The two directions are separate because neither implies the other, and an +assertion of the form "every shipped name exists" cannot be satisfied at all: +the five tables differ between ClickHouse versions, and ``system.events`` +additionally grows only as events fire. Measured on 25.8.33.6, 26.3.28.5 and +26.7.3.19, and on a freshly started 26.7.21.2 — a server that has run nothing — +the shipped/missing split is 59/2 settings, 9/4 server settings, 56/6 gauges, +33/16 events and 32/5 async metrics. A pristine 26.7 is *worse* off than the +used 26.7.3.19 it was measured on. That is the whole reason the inventories are +requests rather than claims; see the module docstring of ``status_introspector``. + +Four claims are pinned specifically because they were false before: + +* ``max_connections`` is **not** a ``system.settings`` row (0 rows on + 26.7.3.19). It is a ``config.xml`` value, so it is read from + ``system.server_settings`` where the server reports it as 4096. +* there is no ``clickhouse.user`` database (26.7.3.19 answers + ``UNKNOWN_DATABASE``), so ``list_users()`` used to return ``[]`` always. It + reads ``system.users`` now and finds the server's own account. +* ``system.processes`` spells the current database ``current_database``; + ``currentDatabase`` is MySQL's name and is not a column here. +* ``system.settings`` carries the description, default, unit and read-only flag + for each row, so none of those are hand-written in the inventory any more. +""" +import inspect +import re + +import pytest + +from rhosocial.activerecord.backend.errors import DatabaseError +from rhosocial.activerecord.backend.impl.clickhouse.introspection.status_introspector import ( + ClickHouseStatusIntrospectorMixin, + SyncClickHouseStatusIntrospector, + AsyncClickHouseStatusIntrospector, + CLICKHOUSE_CONFIG_SETTINGS, + CLICKHOUSE_SERVER_SETTINGS, + CLICKHOUSE_GAUGE_METRICS, + CLICKHOUSE_COUNTER_EVENTS, + CLICKHOUSE_ASYNC_METRICS, + OPEN_CONNECTION_METRICS, +) +from rhosocial.activerecord.backend.introspection.status import StatusCategory + +# (constant, system table, name column) +INVENTORIES = [ + (CLICKHOUSE_CONFIG_SETTINGS, "system.settings", "name"), + (CLICKHOUSE_SERVER_SETTINGS, "system.server_settings", "name"), + (CLICKHOUSE_GAUGE_METRICS, "system.metrics", "metric"), + (CLICKHOUSE_COUNTER_EVENTS, "system.events", "event"), + (CLICKHOUSE_ASYNC_METRICS, "system.asynchronous_metrics", "metric"), +] + +# Names that used to be declared and never returned. +MYSQL_NAMES = [ + "innodb_buffer_pool_size", + "innodb_log_file_size", + "key_buffer_size", + "query_cache_size", + "query_cache_type", + "sql_mode", + "secure_file_priv", + "log_bin", + "binlog_format", + "gtid_mode", + "super_read_only", + "version_comment", + "basedir", + "tmpdir", + "character_set_server", + "collation_server", + "skip_networking", + "skip_name_resolve", + "Threads_connected", + "Threads_running", + "Qcache_hits", + "Qcache_inserts", + "Qcache_lowmem_prunes", + "Com_select", + "Com_insert", + "Com_update", + "Com_delete", + "Com_replace", + "Com_load", + "Bytes_received", + "Bytes_sent", + "Max_used_connections", + "Innodb_buffer_pool_read_requests", + "Innodb_buffer_pool_reads", + "Innodb_buffer_pool_wait_free", + "Innodb_data_reads", + "Innodb_data_writes", + "Innodb_data_read", + "Innodb_data_written", + "Innodb_row_lock_waits", + "Innodb_row_lock_time", + "Innodb_rows_read", + "Innodb_rows_inserted", + "Innodb_rows_updated", + "Innodb_rows_deleted", +] + + +@pytest.fixture +def status(clickhouse_backend_single): + return clickhouse_backend_single.introspector.status + + +def _names_inventory(server_rows, column): + return {row.get(column) for row in server_rows} + + +class TestNothingIsReportedThatTheServerLacks: + """The half that cannot be fudged: no reported name may be fictional. + + This is the direction that matters. A name the server does not publish can + only reach a caller if the module made it up or if the row was fabricated, + and either would be a claim about the server that the server never made. + """ + + @pytest.mark.parametrize( + "table,column,method", + [ + ("system.settings", "name", "list_configuration"), + ("system.server_settings", "name", "list_configuration"), + ("system.metrics", "metric", "list_performance_metrics"), + ("system.events", "event", "list_performance_metrics"), + ("system.asynchronous_metrics", "metric", "list_performance_metrics"), + ], + ) + def test_every_reported_name_is_a_row_of_the_table_it_names( + self, status, clickhouse_backend_single, table, column, method + ): + have = _names_inventory( + clickhouse_backend_single.execute(f"SELECT {column} FROM {table}").data, column + ) + reported = {i.name for i in getattr(status, method)() if i.extra["source"] == table} + fabricated = sorted(reported - have) + assert not fabricated, ( + f"{table} does not have these rows, but they were reported anyway: {fabricated}" + ) + + def test_reported_names_come_from_the_shipped_inventories_not_a_wider_sweep(self, status): + """The inventories must stay the *whole* of what is reported. + + Both accessors build their list by walking the inventory and looking + each name up in the rows, so the shortlist is enforced in Python and + not by the ``WHERE name IN (...)`` clause. Replacing that clause with + an unfiltered read was measured and changes nothing — which is why this + test does not police the SQL and instead pins the outcome: nothing is + reported that the inventories did not ask for. That is the difference + between "this backend's shortlist" and "whatever the table holds", and + it is what keeps ``StatusCategory`` meaningful. + """ + shipped = {n for inventory, _, _ in INVENTORIES for n, _ in inventory} + shipped.update(OPEN_CONNECTION_METRICS) + reported = {i.name for i in status.list_configuration()} + reported.update(i.name for i in status.list_performance_metrics()) + assert reported <= shipped, sorted(reported - shipped) + + def test_every_category_stays_reachable_on_this_server(self, status): + """Category filtering must not collapse to empty on any version. + + The inventories are hand-categorised, so a name that vanishes on an + older release could in principle empty a whole category. Measured on + 25.8.33.6, 26.3.28.5, 26.7.3.19 and a pristine 26.7.21.2: all six + categories keep at least one item, so the filter stays usable. + """ + reported = list(status.list_configuration()) + list(status.list_performance_metrics()) + used = {item.category for item in reported} + assert used == set(StatusCategory), sorted( + c.value for c in set(StatusCategory) - used + ) + + +class TestNothingTheServerHasIsDropped: + """The other direction: an inventory must not empty itself silently. + + Sourcing every value from the row (the module's stated design) is only + useful if the row is actually used. An entry that asks for a name the + server has and then fails to report it is a bug in the assembly, not a fact + about the server — and this is the check that catches it, because the + other direction cannot see it. + """ + + @pytest.mark.parametrize( + "constant,table,column,method", + [ + (CLICKHOUSE_CONFIG_SETTINGS, "system.settings", "name", "list_configuration"), + (CLICKHOUSE_SERVER_SETTINGS, "system.server_settings", "name", "list_configuration"), + (CLICKHOUSE_GAUGE_METRICS, "system.metrics", "metric", "list_performance_metrics"), + (CLICKHOUSE_COUNTER_EVENTS, "system.events", "event", "list_performance_metrics"), + (CLICKHOUSE_ASYNC_METRICS, "system.asynchronous_metrics", "metric", "list_performance_metrics"), + ], + ) + def test_every_shipped_name_the_server_has_is_reported( + self, status, clickhouse_backend_single, constant, table, column, method + ): + have = _names_inventory( + clickhouse_backend_single.execute(f"SELECT {column} FROM {table}").data, column + ) + expected = {n for n, _ in constant if n in have} + assert expected, ( + f"this server shares no name at all with the {table} inventory; " + "the shortlist has drifted away from ClickHouse entirely" + ) + reported = {i.name for i in getattr(status, method)() if i.extra["source"] == table} + dropped = sorted(expected - reported) + assert not dropped, ( + f"{table} has these rows and the inventory asks for them, but they " + f"were not reported: {dropped}" + ) + + def test_no_inventory_repeats_a_name(self): + for inventory, table, _ in INVENTORIES: + names = [n for n, _ in inventory] + assert len(names) == len(set(names)), f"{table} inventory has duplicates" + + def test_open_connection_gauges_are_all_published(self, clickhouse_backend_single): + """These five are summed into ``active_count``, so all five are needed. + + Unlike the curated inventories this set is not a shortlist — dropping a + name would silently under-count — so it is held to the stronger rule + that all of them exist. Measured present on 25.8.33.6, 26.3.28.5 and + 26.7.3.19. + """ + rows = clickhouse_backend_single.execute( + "SELECT metric FROM system.metrics WHERE metric IN (%s)" + % ", ".join(["%s"] * len(OPEN_CONNECTION_METRICS)), + OPEN_CONNECTION_METRICS, + ).data + have = _names_inventory(rows, "metric") + missing = [n for n in OPEN_CONNECTION_METRICS if n not in have] + assert not missing, f"not in system.metrics: {missing}" + + def test_async_inventory_has_no_device_suffixed_names(self): + """Per-CPU / per-disk / per-interface names are generated per host.""" + generated = ("_nbd", "_ram", "_sda", "_sr", "_eth", "_CPU", "_loopback", "_docker", "_veth") + for name, _ in CLICKHOUSE_ASYNC_METRICS: + assert not any(name.endswith(suffix) for suffix in generated), ( + f"{name} carries a per-host device/CPU suffix" + ) + + +class TestNoMysqlNamesAreShipped: + """The names that used to make both inventories return nothing.""" + + def test_none_of_them_is_in_any_inventory(self): + shipped = {n for inventory, _, _ in INVENTORIES for n, _ in inventory} + shipped.update(OPEN_CONNECTION_METRICS) + assert not shipped & set(MYSQL_NAMES) + + @pytest.mark.parametrize("name", MYSQL_NAMES) + def test_mysql_name_is_absent_from_system_tables(self, clickhouse_backend_single, name): + """A MySQL name must not appear in any ClickHouse system table.""" + for table, column in ( + ("system.settings", "name"), + ("system.server_settings", "name"), + ("system.metrics", "metric"), + ("system.events", "event"), + ): + row = clickhouse_backend_single.execute( + f"SELECT count() AS n FROM {table} WHERE {column} = %s", (name,) + ).data + assert row and row[0]["n"] == 0, f"{name!r} unexpectedly exists in {table}" + + +class TestListPerformanceMetrics: + """The function that used to return ``[]`` on every call.""" + + def test_returns_a_non_empty_list(self, status): + items = status.list_performance_metrics() + assert len(items) > 0 + + def test_covers_all_three_metric_tables(self, status): + sources = {item.extra["source"] for item in status.list_performance_metrics()} + assert sources == { + "system.metrics", + "system.events", + "system.asynchronous_metrics", + } + + def test_every_item_has_a_value_and_the_servers_description(self, status): + for item in status.list_performance_metrics(): + assert item.value is not None, item.name + assert item.description, f"{item.name} arrived without a description" + + def test_descriptions_come_from_the_server_not_from_the_module(self, status): + """``Query`` is described by ClickHouse, not by this repository.""" + item = next(i for i in status.list_performance_metrics() if i.name == "Query") + assert item.description == "Number of executing queries" + + def test_inventory_names_are_all_returned(self, status, clickhouse_backend_single): + """Every shipped name this server publishes is returned, per source. + + Qualified by the table, because two of the five inventories can both + contain ``Merge``-shaped names and a bare set comparison would let one + table's row stand in for another's. + """ + for inventory, table, column in INVENTORIES[2:]: + have = _names_inventory( + clickhouse_backend_single.execute(f"SELECT {column} FROM {table}").data, column + ) + returned = { + item.name for item in status.list_performance_metrics() + if item.extra["source"] == table + } + expected = {name for name, _ in inventory if name in have} + missing = sorted(expected - returned) + assert not missing, ( + f"{table} publishes these and the inventory asks for them: {missing}" + ) + + def test_category_filter_is_applied(self, status): + items = status.list_performance_metrics(StatusCategory.STORAGE) + assert items + assert all(item.category == StatusCategory.STORAGE for item in items) + + def test_async_signature_matches_sync(self): + import inspect + + sync_names = { + n for n, _ in inspect.getmembers(SyncClickHouseStatusIntrospector, inspect.isfunction) + } + async_names = { + n for n, _ in inspect.getmembers(AsyncClickHouseStatusIntrospector, inspect.isfunction) + } + assert sync_names == async_names + + +class TestSyncAsyncParity: + """The project's parity rule: same names, same parameters, only await differs.""" + + def _public_methods(self, cls): + return { + name: fn + for name, fn in vars(cls).items() + if not name.startswith("_") and inspect.isfunction(fn) + } + + def test_same_method_names(self): + assert set(self._public_methods(SyncClickHouseStatusIntrospector)) == set( + self._public_methods(AsyncClickHouseStatusIntrospector) + ) + + @pytest.mark.parametrize( + "name", + sorted( + name + for name, fn in vars(SyncClickHouseStatusIntrospector).items() + if not name.startswith("_") and inspect.isfunction(fn) + ), + ) + def test_same_parameters(self, name): + sync = getattr(SyncClickHouseStatusIntrospector, name) + asyn = getattr(AsyncClickHouseStatusIntrospector, name) + assert inspect.iscoroutinefunction(asyn), f"{name} is not a coroutine" + assert not inspect.iscoroutinefunction(sync), f"{name} is unexpectedly a coroutine" + assert list(inspect.signature(sync).parameters) == list(inspect.signature(asyn).parameters) + + def test_the_two_inventories_are_shared_not_duplicated(self): + """Both classes read the same module-level constants.""" + for attr in ( + "list_configuration", + "list_performance_metrics", + "get_connection_info", + "get_storage_info", + "list_databases", + "list_users", + "get_session_info", + "list_processes", + ): + sync = inspect.getsource(getattr(SyncClickHouseStatusIntrospector, attr)) + asyn = inspect.getsource(getattr(AsyncClickHouseStatusIntrospector, attr)) + for const in ( + "CLICKHOUSE_CONFIG_SETTINGS", + "CLICKHOUSE_SERVER_SETTINGS", + "CLICKHOUSE_GAUGE_METRICS", + "CLICKHOUSE_COUNTER_EVENTS", + "CLICKHOUSE_ASYNC_METRICS", + "OPEN_CONNECTION_METRICS", + ): + assert (const in sync) == (const in asyn), f"{attr} differs on {const}" + + +class TestListConfiguration: + def test_reads_both_configuration_tables(self, status): + items = status.list_configuration() + sources = {item.extra["source"] for item in items} + assert sources == {"system.settings", "system.server_settings"} + + def test_every_inventory_name_is_returned(self, status, clickhouse_backend_single): + """Both halves of the split configuration, scoped to their own table. + + ``max_connections`` is the case that makes the scoping necessary: it is + a ``system.server_settings`` row and not a ``system.settings`` row at + all, so a single unqualified name set would pass even if the two queries + were reading the wrong tables. + """ + for inventory, table, column in INVENTORIES[:2]: + have = _names_inventory( + clickhouse_backend_single.execute(f"SELECT {column} FROM {table}").data, column + ) + returned = { + item.name for item in status.list_configuration() + if item.extra["source"] == table + } + expected = {name for name, _ in inventory if name in have} + assert expected, f"the {table} inventory shares no name with this server" + missing = sorted(expected - returned) + assert not missing, ( + f"{table} publishes these and the inventory asks for them: {missing}" + ) + + def test_description_default_and_unit_come_from_the_row(self, status, clickhouse_backend_single): + """``max_threads`` reports whatever the server says, not a fixed string. + + Its default is ``auto(N)`` where N is the number of hardware threads + available to the server — measured ``auto(16)`` on a 16-core host and + ``auto(4)`` on the same 26.3 build pinned to four CPUs with + ``docker run --cpuset-cpus=0-3``. A CI runner has a different core count + again, so the number is not assertable; the *shape* is, and 25.8/26.3 + additionally quote it (``'auto(16)'``, ten characters) where 26.7 does + not. What is pinned here is that all of it came from the row. + """ + row = clickhouse_backend_single.execute( + "SELECT value, default, description, type FROM system.settings WHERE name = %s", + ("max_threads",), + ).data[0] + item = next(i for i in status.list_configuration() if i.name == "max_threads") + assert item.value is not None + assert item.default_value == row["default"] + assert item.extra["type"] == row["type"] == "MaxThreads" + assert re.fullmatch(r"'?auto\(\d+\)'?", item.default_value), item.default_value + assert "query processing threads" in item.description + + def test_time_units_come_from_the_declared_type(self, status): + """``max_execution_time`` is a ``Seconds`` setting on the server.""" + item = next(i for i in status.list_configuration() if i.name == "max_execution_time") + assert item.unit == "Seconds" + + def test_byte_settings_stay_unitless(self, status): + """``UInt64`` does not say bytes, so no unit is claimed for it.""" + item = next( + i for i in status.list_configuration() if i.name == "query_cache_max_size_in_bytes" + ) + assert item.unit is None + + def test_read_only_flag_comes_from_the_server(self, status): + item = next(i for i in status.list_configuration() if i.name == "max_connections") + # config.xml values are fixed unless the server says otherwise. + assert item.is_readonly is True + assert item.is_dynamic is False + + def test_changed_flag_is_recorded(self, status): + for item in status.list_configuration(): + assert isinstance(item.extra["changed"], bool) + + +class TestConnectionInfo: + def test_max_connections_comes_from_server_settings(self, status): + info = status.get_connection_info() + assert info.max_connections is not None + assert info.max_connections > 0 + + def test_max_connections_is_not_a_system_settings_row(self, clickhouse_backend_single): + """Why the lookup had to move: the setting is not in ``system.settings``. + + ``max_connections`` is a genuine ClickHouse name — the old code happened + to have the right word and the wrong table, so the query succeeded and + returned nothing. + """ + absent = clickhouse_backend_single.execute( + "SELECT count() AS n FROM system.settings WHERE name = %s", ("max_connections",) + ).data + assert absent[0]["n"] == 0 + + present = clickhouse_backend_single.execute( + "SELECT value FROM system.server_settings WHERE name = %s", ("max_connections",) + ).data + assert present and int(present[0]["value"]) > 0 + + def test_active_count_sums_the_open_listener_gauges(self, status): + info = status.get_connection_info() + parts = [ + info.extra["http_connections"], + info.extra["tcp_connections"], + info.extra["interserver_connections"], + info.extra["mysql_connections"], + info.extra["postgresql_connections"], + ] + assert all(isinstance(p, int) for p in parts) + assert info.active_count == sum(parts) + assert info.active_count >= 1 + + def test_idle_count_is_none_and_says_why(self, status): + info = status.get_connection_info() + assert info.idle_count is None + assert "Threads_connected" in info.extra["note"] + + +class TestListUsers: + def test_reads_system_users(self, status, clickhouse_backend_single): + expected = { + row["name"] + for row in clickhouse_backend_single.execute("SELECT name FROM system.users").data + } + assert {u.name for u in status.list_users()} == expected + assert expected, "the scenario server reports no users at all" + + def test_the_clickhouse_user_database_does_not_exist(self, clickhouse_backend_single): + """The source the old implementation used, and why it always returned [].""" + with pytest.raises(DatabaseError): + clickhouse_backend_single.execute("SELECT User, Host FROM clickhouse.user") + + def test_superuser_is_derived_from_grants(self, status): + for user in status.list_users(): + types = set(user.extra["access_types"]) + expected = all(t in types for t in user.extra["superuser_test"]) + assert user.is_superuser is expected + assert user.extra["superuser_test"], "the superuser test set must be recorded" + + def test_auth_types_are_reported(self, status): + assert any(u.extra["auth_type"] for u in status.list_users()) + + +class TestListProcesses: + def test_database_column_is_read_from_current_database(self, status): + procs = status.list_processes() + assert procs, "system.processes always has at least the querying session" + assert procs[0].database is not None + + def test_currentDatabase_is_not_a_column(self, clickhouse_backend_single): + """``system.processes`` names it ``current_database``. + + Note the trap: ``SELECT currentDatabase FROM system.processes`` does *not* + fail, because ClickHouse resolves the bare word to its ``currentDatabase()`` + function and returns that instead. The column list is the only thing that + settles it. + """ + columns = { + row["name"] + for row in clickhouse_backend_single.execute( + "SELECT name FROM system.columns WHERE database = 'system' AND table = 'processes'" + ).data + } + assert "current_database" in columns + assert "currentDatabase" not in columns + + +class TestStorageInfo: + def test_free_space_is_populated_from_system_disks(self, status): + info = status.get_storage_info() + assert info.free_space_bytes is not None + assert info.free_space_bytes > 0 + + def test_all_disks_are_reported(self, status): + info = status.get_storage_info() + assert info.extra["disks"] + for disk in info.extra["disks"]: + assert disk["name"] and disk["path"] + + def test_total_size_is_the_configured_database(self, status, clickhouse_backend_single): + expected = clickhouse_backend_single.execute( + "SELECT sum(total_bytes) AS n FROM system.tables WHERE database = %s", + (clickhouse_backend_single.config.database,), + ).data[0]["n"] + assert status.get_storage_info().total_size_bytes == expected + + +class TestOverview: + def test_overview_is_populated(self, status): + ov = status.get_overview() + assert ov.server_vendor == "ClickHouse" + assert ov.server_version + assert ov.configuration + assert ov.performance + assert ov.databases + assert ov.users + assert ov.processes + assert ov.connections.max_connections + + def test_mysql_only_slots_are_left_at_their_default(self, status): + """``innodb`` / ``binary_log`` / ``slow_query`` are core fields this + backend never fills; they must be ``None``, not fabricated.""" + ov = status.get_overview() + assert ov.innodb is None + assert ov.binary_log is None + assert ov.slow_query is None + + def test_the_mixin_no_longer_imports_an_innodb_type(self): + import rhosocial.activerecord.backend.impl.clickhouse.introspection.status_introspector as mod + + source_names = [n for n in dir(mod) if "InnoDB" in n or "BinaryLog" in n or "SlowQuery" in n] + assert source_names == [] + assert "innodb" not in mod.ClickHouseStatusIntrospectorMixin._build_server_overview.__code__.co_varnames + + def test_to_dict_is_json_ready(self, status): + import json + + payload = status.get_overview().to_dict() + json.dumps(payload) # must not raise + assert payload["innodb"] is None + # to_dict() already unwraps the enum to its value. + assert payload["performance"][0]["category"] in { + "configuration", + "performance", + "connection", + "storage", + "security", + "replication", + } + + +class TestDatabaseTableCounts: + """The counts are ``information_schema``'s, so an empty database says zero. + + A fresh ClickHouse container has no tables anywhere except ``system`` and + ``information_schema``; the scenario creates ``test_db`` but nothing puts a + table *in* it before this runs. Measured on 25.8.33.6, 26.3.28.5 and + 26.7.3.19: ``default`` holds 0 tables on all three. + """ + + def test_the_database_list_is_the_servers_own(self, status, clickhouse_backend_single): + """``list_databases()`` names what the server has, with a real count each. + + Compared against ``system.databases`` rather than a literal, so a server + that has an extra database still passes — and so the assertion still + fails if a name comes back ``None`` or a count comes back missing, which + is what the MySQL-shaped parsers used to produce. + """ + expected = { + row["name"] + for row in clickhouse_backend_single.execute("SELECT name FROM system.databases").data + } + dbs = {db.name: db for db in status.list_databases()} + assert set(dbs) == expected + assert "default" in dbs + assert all(isinstance(db.table_count, int) and db.table_count >= 0 for db in dbs.values()) + + def test_counts_match_information_schema_for_a_known_database( + self, status, clickhouse_backend_single + ): + """Pinned against a database whose contents this test controls. + + Asserting against ``information_schema`` itself rather than a literal + number is what makes the check meaningful on a server whose tables come + and go: it still fails if the count is read from the wrong column or the + wrong table, and it cannot fail merely because a database is empty. + """ + database = clickhouse_backend_single.config.database + row = clickhouse_backend_single.execute( + "SELECT count() AS n FROM information_schema.TABLES " + "WHERE table_schema = %s AND table_type = 'BASE TABLE'", + (database,), + ).data[0] + dbs = {db.name: db for db in status.list_databases()} + assert database in dbs, f"{database} is the configured database and must be listed" + assert dbs[database].table_count == row["n"] + + def test_view_count_is_not_zero_for_information_schema(self, status): + """``information_schema`` really does have views, so this is not a zero.""" + dbs = {db.name: db for db in status.list_databases()} + assert dbs["information_schema"].view_count > 0 + + +class TestUnitDerivation: + """``_setting_unit`` is pure, so it is pinned without a server.""" + + @pytest.mark.parametrize( + "declared,expected", + [ + ("Seconds", "Seconds"), + ("Milliseconds", "Milliseconds"), + ("Microseconds", "Microseconds"), + ("Nanoseconds", "Nanoseconds"), + ("UInt64", None), + ("NonZeroUInt64", None), + ("Bool", None), + ("MaxThreads", None), + ("Float", None), + (None, None), + ], + ) + def test_only_time_types_become_units(self, declared, expected): + assert ClickHouseStatusIntrospectorMixin._setting_unit(declared) == expected \ No newline at end of file diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/protocol/test_protocol_conformance.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/protocol/test_protocol_conformance.py index 41bff08..92590a8 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/protocol/test_protocol_conformance.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/protocol/test_protocol_conformance.py @@ -128,7 +128,7 @@ def get_own_protocol_methods(proto: type) -> set: dialect_protocols.CreateTableUsingTemplateSupport, dialect_protocols.CreateTriggerSupport, dialect_protocols.CreateViewSupport, - dialect_protocols.DateTimeSupport, + dialect_protocols.TimestampSupport, dialect_protocols.DqlOrderSupport, dialect_protocols.DropIndexSupport, dialect_protocols.DropTableSupport, @@ -147,13 +147,19 @@ def get_own_protocol_methods(proto: type) -> set: # probes, and ClickHouse answers supports_identity_column() False -- its # server rejects the clause with Code 62 (syntax error at GENERATED). dialect_protocols.IdentityColumnSupport, + # The column-type table: `ClickHouseColumnTypeMixin` answers + # `suggested_column_types()` for all eighteen common Python types and + # `suggested_extra_column_types()` with nothing of its own, and the + # dialect composes the mixin. The table is the model layer's source for + # which column class a field's annotation resolves to, so ClickHouse + # declares it rather than leaving it for another dialect to answer. + dialect_protocols.ColumnTypeSupport, # ClickHouse-specific protocols clickhouse_protocols.ClickHouseDMLOperationSupport, clickhouse_protocols.ClickHouseTriggerSupport, clickhouse_protocols.ClickHouseTableSupport, clickhouse_protocols.ClickHouseJSONFunctionSupport, clickhouse_protocols.ClickHouseSpatialSupport, - clickhouse_protocols.ClickHouseVectorSupport, clickhouse_protocols.ClickHouseFullTextSearchSupport, clickhouse_protocols.ClickHouseLockingSupport, clickhouse_protocols.ClickHouseModifyColumnSupport, @@ -194,6 +200,11 @@ def test_implements_protocol(self, dialect, protocol): # ever satisfies one by accident, the negative test fails and forces a conscious # decision (move to CLICKHOUSE_PROTOCOLS or revert). CLICKHOUSE_NOT_IMPLEMENTED = [ + # UUID value expressions (generation / nil-max constants / cast) are not + # implemented yet on this dialect. Listed here so the omission is a + # recorded decision rather than a gap; move it to the implemented list + # when the mixin lands. + dialect_protocols.UUIDSupport, # --- Intentional non-support --- # ClickHouse namespaces objects with databases only. There is no inner # schema, so it qualifies a name with the catalog slot and reports a @@ -422,11 +433,9 @@ class TestClickHouseExpressionDialectSeparation: ("ClickHouseJSONObjectExpression", "format_json_object"), ("ClickHouseJSONArrayExpression", "format_json_array"), ("ClickHouseJSONContainsExpression", "format_json_contains"), - ("ClickHouseAddPartitionExpression", "format_add_partition_statement"), ("ClickHouseDropPartitionExpression", "format_drop_partition_statement"), - ("ClickHouseTruncatePartitionExpression", "format_truncate_partition_statement"), - ("ClickHouseReorganizePartitionExpression", "format_reorganize_partition_statement"), - ("ClickHouseExchangePartitionExpression", "format_exchange_partition_statement"), + ("ClickHouseDetachPartitionExpression", "format_detach_partition_statement"), + ("ClickHouseAttachPartitionExpression", "format_attach_partition_statement"), ("ClickHouseRenameTableExpression", "format_rename_table_statement"), ] @@ -467,7 +476,6 @@ def test_expression_delegates_to_dialect(self, expr_name, format_method): (clickhouse_protocols.ClickHouseTableSupport, clickhouse_mixins.ClickHouseTableMixin), (clickhouse_protocols.ClickHouseJSONFunctionSupport, clickhouse_mixins.ClickHouseJSONFunctionMixin), (clickhouse_protocols.ClickHouseSpatialSupport, clickhouse_mixins.ClickHouseSpatialMixin), - (clickhouse_protocols.ClickHouseVectorSupport, clickhouse_mixins.ClickHouseVectorMixin), (clickhouse_protocols.ClickHouseFullTextSearchSupport, clickhouse_mixins.ClickHouseFullTextSearchMixin), (clickhouse_protocols.ClickHouseLockingSupport, clickhouse_mixins.ClickHouseLockingMixin), (clickhouse_protocols.ClickHouseModifyColumnSupport, clickhouse_mixins.ClickHouseModifyColumnMixin), diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/test_expression_roundtrip_all.py index 25108b9..e49d108 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/test_expression_roundtrip_all.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/test_expression_roundtrip_all.py @@ -32,8 +32,15 @@ * **renders** -- all three encodings must restore byte-identical SQL *and* byte-identical bind parameters. -* ``UnsupportedFeatureError`` -- this dialect does not model the feature. - Asserted as exactly that type, so a different failure cannot hide behind it. +* ``UnsupportedFeatureError`` from a probe inside a formatter -- this dialect + does not model the feature. Asserted as exactly that type, so a different + failure cannot hide behind it. +* ``UnsupportedFeatureError`` from the dispatch -- the dialect's base list + names no ``format_*`` method for the class at all, which is a wiring fact + about this backend rather than a capability answer. The method the dispatch + names must be a row in :data:`UNMODELLED_FORMATTERS`, so the gap is a + decision with a reason rather than an omission; see + :func:`_dispatched_formatter`. * a member of :data:`LEGITIMATE_NON_RENDERS` -- a class that cannot render for a reason belonging to its own tree. Each entry pins the exception type *and* a message fragment, so a class that starts failing differently fails here. @@ -52,6 +59,24 @@ :func:`test_pinned_non_render_really_does_not_render` is what makes the entry load-bearing in both directions. +The same incident is also why the blanket branch itself was split in two +======================================================================= + +A probe inside a formatter and the dispatch naming a missing formatter raise the +same type, and for a long time this file allowed both identically -- which let a +class whose formatter this dialect never mixed in read as "ClickHouse lacks the +feature" and pass. Core turning ``trim``/``lpad``/``rpad``/``repeat`` into +dedicated nodes made the hole load-bearing: a backend shipping no formatter for +one of them would answer every ``.lpad(...)`` with that refusal, and nothing here +could tell it apart from a genuine "not on this engine". The exception itself +records who is refusing: a probe names the feature, while the dispatch frames +the formatting method it could not find as ``the '' statement`` -- see +:func:`_dispatched_formatter`. Only the dispatch's refusal is a wiring fact +about this backend, so only it must be accounted for: +:meth:`TestMatrixIntegrity.test_unmodelled_formatter_list_is_exact` pins +:data:`UNMODELLED_FORMATTERS` in both directions, and a method nobody listed +fails the run. + And what happens when a class cannot be constructed =================================================== @@ -159,7 +184,6 @@ maintenance, materialized_view, partition, - partition_lifecycle, rename_table, routine, ) @@ -290,17 +314,6 @@ def _integer_column(dialect, name="col"): return ddl_table.ColumnDefinition(dialect, name, IntegerType(dialect)) -def _partition_definition(dialect): - """A named ClickHouse partition with a bound upper limit.""" - return partition.ClickHousePartitionDefinition( - name="p2024", less_than=[Literal(dialect, 2025)] - ) - - -def _partition_keys(dialect): - return [Column(dialect, "id")] - - def register_specials(): """Replace every introspective guess that would cost a real assertion.""" # -- the object a statement names, and the source that reads it ---------- @@ -480,124 +493,20 @@ def register_specials(): ) # -- ClickHouse partitions ---------------------------------------------- - # ``keys`` is keyword-only behind a defaulted positional for the ``*By*`` - # family, so the introspective constructor skips it and the clause declares - # no partitioning at all. - for _name in ( - "ClickHousePartitionByRange", - "ClickHousePartitionByRangeColumns", - "ClickHousePartitionByList", - "ClickHousePartitionByListColumns", - ): - register_special_constructor( - f"partition.{_name}", - lambda d, _n=_name: getattr(partition, _n)(d, keys=_partition_keys(d)), - ) - register_special_constructor( - "partition.ClickHousePartitionByHash", - lambda d: partition.ClickHousePartitionByHash( - d, keys=_partition_keys(d), partitions_count=4 - ), - ) - register_special_constructor( - "partition.ClickHousePartitionByKey", - lambda d: partition.ClickHousePartitionByKey(d, keys=_partition_keys(d)), - ) - register_special_constructor( - "partition.ClickHouseSubpartitionClause", - lambda d: partition.ClickHouseSubpartitionClause( - d, partition.ClickHouseSubpartitionStrategy.HASH, count=2 - ), - ) - register_special_constructor( - "partition.ClickHousePartitionClause", - lambda d: partition.ClickHousePartitionClause( - d, partition.ClickHousePartitionStrategy.RANGE, keys=_partition_keys(d) - ), - ) - register_special_constructor( - "partition.ClickHousePartitionNameListExpression", - lambda d: partition.ClickHousePartitionNameListExpression(d, ["p2024"]), - ) - register_special_constructor( - "partition.ClickHousePartitionValue", - lambda d: partition.ClickHousePartitionValue(d, 1), - ) - register_special_constructor( - "partition.ClickHouseGetPartitionsExpression", - lambda d: partition.ClickHouseGetPartitionsExpression(d, _table(d)), - ) - # The partition maintenance statements share one shape: a base table and a - # list of partition names. + # The MySQL-style declarative partitioning family was deleted outright: + # ClickHouse declares partitioning as an arbitrary expression in CREATE + # TABLE, not as PARTITION ... VALUES statements. What remains are the + # three system.parts-addressed statements, each taking the table name + # and the partition id, and each rendering through one shared base. for _name in ( "ClickHouseDropPartitionExpression", - "ClickHouseTruncatePartitionExpression", - "ClickHouseRebuildPartitionExpression", - "ClickHouseRepairPartitionExpression", - "ClickHouseAnalyzePartitionExpression", - "ClickHouseCheckPartitionExpression", - "ClickHouseOptimizePartitionExpression", + "ClickHouseDetachPartitionExpression", + "ClickHouseAttachPartitionExpression", ): register_special_constructor( f"partition.{_name}", - lambda d, _n=_name: getattr(partition, _n)( - d, table=_table(d), partitions=["p2024"] - ), + lambda d, _n=_name: getattr(partition, _n)(d, "t", "p2024"), ) - register_special_constructor( - "partition.ClickHouseCoalescePartitionExpression", - lambda d: partition.ClickHouseCoalescePartitionExpression( - d, table=_table(d), count=2 - ), - ) - register_special_constructor( - "partition.ClickHouseRemovePartitioningExpression", - lambda d: partition.ClickHouseRemovePartitioningExpression(d, table=_table(d)), - ) - register_special_constructor( - "partition.ClickHouseAddPartitionExpression", - lambda d: partition.ClickHouseAddPartitionExpression( - d, table=_table(d), partitions=[_partition_definition(d)] - ), - ) - register_special_constructor( - "partition.ClickHouseReorganizePartitionExpression", - lambda d: partition.ClickHouseReorganizePartitionExpression( - d, table=_table(d), partition="p2024", into=[_partition_definition(d)] - ), - ) - register_special_constructor( - "partition.ClickHouseExchangePartitionExpression", - lambda d: partition.ClickHouseExchangePartitionExpression( - d, table=_table(d), partition="p2024", exchange_table=Table(d, "other") - ), - ) - - # -- ClickHouse partition helpers --------------------------------------- - register_special_constructor( - "partition_lifecycle.ClickHouseAddPartitionHelper", - lambda d: partition_lifecycle.ClickHouseAddPartitionHelper( - d, table=_table(d), partition_values=[1] - ), - ) - register_special_constructor( - "partition_lifecycle.ClickHouseDropOldestPartitionHelper", - lambda d: partition_lifecycle.ClickHouseDropOldestPartitionHelper( - d, table=_table(d), partition_names=["p0"] - ), - ) - register_special_constructor( - "partition_lifecycle.ClickHouseAddSubpartitionHelper", - lambda d: partition_lifecycle.ClickHouseAddSubpartitionHelper( - d, table=_table(d), partition_name="p0", less_than=[Literal(d, 2)] - ), - ) - register_special_constructor( - "partition_lifecycle.ClickHouseCoalescePartitionHelper", - lambda d: partition_lifecycle.ClickHouseCoalescePartitionHelper( - d, table=_table(d), target_count=1, current_count=3 - ), - ) # -- ClickHouse maintenance, routines, materialized views ---------------- for _name in ( @@ -674,6 +583,20 @@ def register_specials(): "rename_table.ClickHouseRenameTableExpression", lambda d: rename_table.ClickHouseRenameTableExpression(d, [("a", "b")]), ) + # CUSTOM. Its ``raw`` slot exists to carry the SQL type name and defaults + # to the empty string, which the filler skips and the class refuses while + # constructing. Handed a real name it builds -- and then the dialect + # refuses it, which is what the existing LEGITIMATE_NON_RENDERS pin below + # asserts. A constructor rather than a skip, so the pin has something to + # assert against. + def _custom_type(dialect): + from rhosocial.activerecord.backend.expression.types.custom import ( + CustomType, + ) + + return CustomType(dialect, "String") + + register_special_constructor("types.custom.CustomType", _custom_type) register_specials() @@ -720,6 +643,13 @@ def register_specials(): # positional, so the introspective constructor skips it and the type declares # no members. "rhosocial.activerecord.backend.expression.types.enum_.EnumType", + # The three UUID nodes refuse in __init__, not in to_sql(): ClickHouse + # spells no UUID SQL, so the answer is the same for every argument and no + # registered constructor can change it. Measured with a valid ``which`` -- + # an invalid one raises ValueError first and hides the dialect's answer. + "rhosocial.activerecord.backend.expression.uuid.UUIDCastExpression", + "rhosocial.activerecord.backend.expression.uuid.UUIDConstantExpression", + "rhosocial.activerecord.backend.expression.uuid.UUIDGenerationExpression", ) # Four XML classes used to be entries here, and retiring them is what the pin # demands rather than a loss of coverage: each named a gap that no longer exists. @@ -785,6 +715,14 @@ class on purpose: if a class changed the formatter it dispatches on, the NotImplementedError, _NO_FORMATTER, ), + # The shared base of the three ``ALTER TABLE ... PARTITION ID`` clauses. + # Each concrete subclass declares its own ``format_method``; the base names + # the convention and refuses, which is the same answer as the category + # bases above rather than a construction gap. + "rhosocial.activerecord.backend.impl.clickhouse.expression.partition._ClickHousePartitionIdExpression": ( + NotImplementedError, + "must declare its format_method property", + ), # An expression-category base whose concrete members each name their own # formatter: an ALTER TABLE action, an INSERT row source, a transaction step, # and the roots of the object, type and routine trees. @@ -1046,21 +984,134 @@ class on purpose: if a class changed the formatter it dispatches on, the TypeError, "does not support the generic type 'timestamptz'", ), - "rhosocial.activerecord.backend.expression.types.integer.IntType": ( - TypeError, - "does not support the generic type 'int'", - ), "rhosocial.activerecord.backend.expression.types.json_.JsonBType": ( TypeError, "does not support the generic type 'jsonb'", ), - "rhosocial.activerecord.backend.expression.types.uuid_.UUIDType": ( + "rhosocial.activerecord.backend.expression.types.xml_.XmlType": ( TypeError, - "does not support the generic type 'uuid'", + "does not support the generic type 'xml'", + ), +} + + +# --------------------------------------------------------------------------- +# The dispatch gap: a missing formatter is a decision, not an omission +# --------------------------------------------------------------------------- +# +# ``to_sql()`` reports a dialect that names no formatter for a class through +# the same ``UnsupportedFeatureError`` a capability probe uses, so the type +# alone cannot say "this backend never wired the feature in". The frame the +# dispatch puts around the method name can, and it is the only place in the +# tree that opens it -- see :func:`_dispatched_formatter`. + +#: The two ends of the frame ``to_sql()`` puts around a formatting method name +#: in the ``feature_name`` of the ``UnsupportedFeatureError`` it raises for a +#: dialect that has no such formatter. +_DISPATCH_FEATURE_PREFIX = "the '" +_DISPATCH_FEATURE_SUFFIX = "' statement" + + +def _dispatched_formatter(exc): + """The formatting method *exc* says the dialect does not have, or ``None``. + + ``None`` means *exc* is not the dispatch reporting a missing formatter -- + it is a probe inside a formatter that refused a feature, which is the + ``"unsupported"`` branch and needs no row anywhere. + + Args: + exc: An :class:`UnsupportedFeatureError` raised out of ``to_sql()``. + + Returns: + The method name out of the ``feature_name`` frame, or ``None`` when + *exc* came from a probe rather than from the dispatch. + """ + feature = exc.feature_name + if not feature.startswith(_DISPATCH_FEATURE_PREFIX): + return None + if not feature.endswith(_DISPATCH_FEATURE_SUFFIX): + return None + method = feature[len(_DISPATCH_FEATURE_PREFIX) : len(feature) - len(_DISPATCH_FEATURE_SUFFIX)] + return method if method else None + + +#: Formatters ClickHouse declares no implementation of, grouped by the feature +#: that is absent, each group naming every method the matrix's classes need. +#: +#: A class needing one of these fails with ``UnsupportedFeatureError`` naming +#: the method it wanted, and :func:`assert_sql_roundtrip_classified` asserts +#: that method is a row here. :meth:`TestMatrixIntegrity.\ +#: test_unmodelled_formatter_list_is_exact` pins the table in both directions: +#: a formatter that starts existing moves its classes out of the table and the +#: pin fails until the entry goes, and a class that starts needing a method +#: outside the table fails outright instead of passing as "unsupported". +#: +#: These are not defects: they are features ClickHouse does not have, which the +#: shared renderer reports by naming the method it could not find. The grouping +#: is the point -- "31 methods" says nothing, "no SQL/XML, no property-graph +#: DDL, no schema, no sequence, no COMMENT ON, no ILIKE, no PIVOT" says what +#: the dialect is. ClickHouse does spell the new pad/repeat/trim nodes -- LPAD, +#: RPAD and REPEAT natively, and TRIM through ClickHouseTrimMixin's +#: ``trimBoth``/``trimLeft``/``trimRight`` -- so none of the four appears here. +UNMODELLED_FORMATTERS = { + "SQL/XML (ClickHouse mixes in no SQL/XML formatter at all)": ( + "format_xmlagg_expression", + "format_xmlattributes_expression", + "format_xmlcomment_expression", + "format_xmlconcat_expression", + "format_xmlelement_expression", + "format_xmlexists_expression", + "format_xmlforest_expression", + "format_xmlpi_expression", + "format_xmlparse_expression", + "format_xmlquery_expression", + "format_xmlroot_expression", + "format_xmlserialize_expression", + "format_xmltable_expression", + ), + "SQL/PGQ (ClickHouse mixes in GraphMixin, whose MATCH formatters exist " + "and refuse; the GRAPH_TABLE half lives in GraphTableMixin, which is not " + "mixed in)": ( + "format_graph_table_expression", + "format_vertex_table", + "format_edge_table", + "format_graph_columns_clause", + "format_table_properties_clause", + "format_create_property_graph_statement", + "format_drop_property_graph_statement", + "format_alter_property_graph_statement", ), + "COMMENT ON (the dialect mixes in no comment-DDL mixin; ClickHouse " + "comments ride on table and column definitions)": ("format_comment_statement",), + "SCHEMA (ClickHouse has a database and no inner schema, so CREATE/DROP " + "SCHEMA is not in its grammar at all)": ( + "format_create_schema_statement", + "format_drop_schema_statement", + ), + "SEQUENCE (ClickHouse has no sequence object; naming a sequence is a " + "separate capability and is untouched -- SequenceNameMixin stays)": ( + "format_create_sequence_statement", + "format_drop_sequence_statement", + "format_alter_sequence_statement", + ), + "ILIKE (the dialect mixes in no ILIKE mixin; case-insensitive matching " + "is ClickHouse's positionCaseInsensitive)": ("format_ilike_expression",), + "the core PIVOT / UNPIVOT expressions (the dialect mixes in no pivot " + "mixin)": ( + "format_pivot_expression", + "format_unpivot_expression", + ), + "the TableSource root (every concrete row source overrides its " + "formatter; the root carries nothing but an alias and is never " + "rendered)": ("format_table_source",), } +def _unmodelled_methods(): + """The flat set of formatter names in :data:`UNMODELLED_FORMATTERS`.""" + return {method for methods in UNMODELLED_FORMATTERS.values() for method in methods} + + # --------------------------------------------------------------------------- # The local SQL assertion: classify the outcome instead of swallowing it # --------------------------------------------------------------------------- @@ -1073,18 +1124,23 @@ def assert_sql_roundtrip_classified(fqn, instance, dialect): * **renders** -- all three encodings must restore byte-identical SQL *and* byte-identical bind parameters. - * ``UnsupportedFeatureError`` -- the dialect does not model the feature. - Asserted as exactly that type, so a formatter raising it for an unrelated - reason is still visible as that type rather than as a pass. + * ``UnsupportedFeatureError`` from a probe inside a formatter -- the + dialect does not model the feature. Asserted as exactly that type, so a + formatter raising it for an unrelated reason is still visible as that + type rather than as a pass. + * ``UnsupportedFeatureError`` from the dispatch -- the dialect's base list + names no ``format_*`` method for the class at all, which is a wiring + fact about this backend rather than a capability answer. The method must + be in :data:`UNMODELLED_FORMATTERS`; the table entry is the reason. * a member of :data:`LEGITIMATE_NON_RENDERS` -- unrenderable by design, asserted as its exact type *and* message fragment. * **anything else** -- a failure naming the class and the exception. - The second and third overlap: a pinned entry may itself raise - ``UnsupportedFeatureError``, in which case the blanket branch above takes it - and this function does not look at the pin. That is fine -- the pin's job for - those classes is to name the specific missing formatter, and that is asserted - by :meth:`TestMatrixIntegrity.test_pinned_non_render_really_does_not_render`. + The first, third and fourth overlap: a pinned entry may itself raise + ``UnsupportedFeatureError``, in which case the dispatch branch above takes + it and this function does not look at the pin. That is fine -- the pin's + job for those classes is to name the specific missing formatter, and that + is asserted by :meth:`TestMatrixIntegrity.test_pinned_non_render_really_does_not_render`. Returns: A short string naming the branch taken, so a caller can report the @@ -1098,6 +1154,17 @@ def assert_sql_roundtrip_classified(fqn, instance, dialect): expected_sql, expected_params = instance.to_sql() except UnsupportedFeatureError as exc: assert type(exc) is UnsupportedFeatureError, fqn + method = _dispatched_formatter(exc) + if method is not None: + assert method in _unmodelled_methods(), ( + f"{fqn}: to_sql() reported that {exc.dialect_name} declares no " + f"{method!r}, which is not in UNMODELLED_FORMATTERS.\n" + f" Either ClickHouse now needs that formatter -- in which case " + f"the class should render and this entry should go -- or the " + f"feature is absent and the method belongs in that table with " + f"its reason." + ) + return "unmodelled" return "unsupported" except Exception as exc: if fqn not in LEGITIMATE_NON_RENDERS: @@ -1506,6 +1573,60 @@ def test_pinned_non_render_really_does_not_render(self, clickhouse_matrix_dialec f"{exc_info.value}" ) + def test_unmodelled_formatter_list_is_exact(self, clickhouse_matrix_dialect): + """Pin the unmodelled-formatter table against what the dialect lacks. + + Observed rather than assumed: for each class the matrix covers, either + it renders or it fails, and every dispatch refusal is attributed to the + method it named -- including the classes pinned in + :data:`LEGITIMATE_NON_RENDERS`, whose dispatch failures those pins + record class by class. Then the table is compared with the set + observed, in both directions, so: + + * a method in the table that ClickHouse now implements fails here, + because its classes render and are no longer attributed to it -- + which is the moment to delete the entry rather than leave a lie in + the table; + * a class that starts needing a method outside the table fails the + per-class assertion in :func:`assert_sql_roundtrip_classified` + instead of being absorbed into "unsupported". + + This is the regression test for the pad/trim incident. Core turned + ``trim``/``lpad``/``rpad``/``repeat`` into dedicated nodes; ClickHouse + spells all four -- LPAD, RPAD and REPEAT natively, TRIM through + ClickHouseTrimMixin -- so none of the four is a gap here, and this test + is what would have said so loudly if a fifth node of that family had + arrived without a ClickHouse formatter. + """ + observed = set() + for fqn in sorted(REGISTERED): + instance, source = make_instance(REGISTERED[fqn], clickhouse_matrix_dialect) + if instance is None: + continue + try: + instance.to_sql() + except UnsupportedFeatureError as exc: + method = _dispatched_formatter(exc) + if method is not None: + observed.add(method) + continue + except Exception: + continue + + declared = _unmodelled_methods() + assert not (observed - declared), ( + "classes need formatters that are not in UNMODELLED_FORMATTERS: " + f"{sorted(observed - declared)}. Add each with the feature that is " + f"absent and why -- or mix the formatter in, in which case the " + f"classes render and the entry is not needed." + ) + assert not (declared - observed), ( + "UNMODELLED_FORMATTERS names formatters no class actually needs: " + f"{sorted(declared - observed)}. ClickHouse may have gained one of " + f"these, in which case the classes needing it now render and the " + f"entry should go." + ) + def test_matrix_covers_both_packages(self): """The matrix covers every concrete class in the two packages. @@ -1567,7 +1688,11 @@ def test_coverage_report(self, clickhouse_matrix_dialect): f"pinned-unconstructible, {len(LEGITIMATE_NON_RENDERS)} " f"pinned-non-render; rendered={branches.get('rendered', 0)}, " f"unsupported={branches.get('unsupported', 0)}, " + f"unmodelled={branches.get('unmodelled', 0)} (pinned in " + f"UNMODELLED_FORMATTERS), " f"non-render={branches.get('non-render', 0)}" ) for fqn in UNCONSTRUCTIBLE: print(f" not constructible: {fqn}") + for feature, methods in sorted(UNMODELLED_FORMATTERS.items()): + print(f" not modelled: {feature} ({len(methods)} formatters)") diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_clickhouse_type_protocol.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_clickhouse_type_protocol.py index 92950fd..981f623 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_clickhouse_type_protocol.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_clickhouse_type_protocol.py @@ -3,12 +3,34 @@ from __future__ import annotations +import inspect import re import pytest -from rhosocial.activerecord.backend.expression.types import DataType +from rhosocial.activerecord.backend.expression.types import ( + ArrayType, + BigIntType, + BinaryType, + BooleanType, + CharType, + DataType, + DateType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + JsonType, + RealType, + SmallIntType, + TextType, + TimestampType, + TinyIntType, + UUIDType, + VarCharType, +) from rhosocial.activerecord.backend.impl.clickhouse.dialect import ClickHouseDialect +from rhosocial.activerecord.backend.impl.clickhouse.expression import types as clickhouse_types from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( ClickHouseAggregateFunctionType, ClickHouseArrayType, @@ -57,6 +79,10 @@ from rhosocial.activerecord.backend.impl.clickhouse.mixins.types import ( ClickHouseTypeSupportMixin, ) +from rhosocial.activerecord.backend.dialect.protocols import DataTypeSupport +from rhosocial.activerecord.backend.impl.clickhouse.protocols import ( + ClickHouseTypeSupport, +) # ── W1: Name prefix enforcement ─────────────────────────────────────── @@ -125,10 +151,10 @@ def test_constructor_rejects_dialect_options(self): -# ── W1: _type_params replaces hand-written __eq__/__hash__ ──────────── +# ── W1: PARAMETERS replaces hand-written __eq__/__hash__ ──────────── class TestTypeParamsSemantics: - def test_decimal_equality_via_type_params(self): + def test_decimal_equality_via_declared_identity(self): a = ClickHouseDecimalType(precision=10, scale=2) b = ClickHouseDecimalType(precision=10, scale=2) c = ClickHouseDecimalType(precision=10, scale=3) @@ -220,8 +246,8 @@ def test_type_is_value_object(self): # ── W1: No hand-written __eq__/__hash__ on subclasses ───────────────── class TestNoHandWrittenEqHash: - """Verify that parameterized types use _type_params() and inherit - __eq__/__hash__ from DataType (which uses _type_params).""" + """Verify that parameterized types declare PARAMETERS and inherit + __eq__/__hash__ from DataType (which reads PARAMETERS).""" PARAMETERIZED_TYPES = [ ClickHouseDecimalType(precision=10, scale=2), @@ -245,13 +271,13 @@ class TestNoHandWrittenEqHash: def test_no_custom_eq(self): for instance in self.PARAMETERIZED_TYPES: assert "__eq__" not in type(instance).__dict__, ( - f"{type(instance).__name__} defines __eq__ — should use _type_params()" + f"{type(instance).__name__} defines __eq__ — should declare PARAMETERS" ) def test_no_custom_hash(self): for instance in self.PARAMETERIZED_TYPES: assert "__hash__" not in type(instance).__dict__, ( - f"{type(instance).__name__} defines __hash__ — should use _type_params()" + f"{type(instance).__name__} defines __hash__ — should declare PARAMETERS" ) @@ -313,11 +339,22 @@ def test_includes_core_types(self): "varchar", "char", "text", "boolean", "date", "datetime", "timestamp", "time", "float", "double", "real", "decimal", - "json", "blob", "array", + "json", "blob", "array", "uuid", ] for name in core_names: assert name in supported, f"Core type {name!r} missing from supports_data_types()" + def test_core_names_map_to_the_core_classes(self): + """The value is the class whose ``name`` is the key, so + ``supports_data_types()['integer'] is IntegerType``.""" + dialect = ClickHouseDialect() + supported = dialect.supports_data_types() + assert supported["integer"] is IntegerType + assert supported["bigint"] is BigIntType + assert supported["uuid"] is UUIDType + assert supported["clickhouse_int8"] is ClickHouseInt8Type + assert supported["clickhouse_timestamp"] is ClickHouseDateTimeType + def test_values_are_data_type_subclasses(self): dialect = ClickHouseDialect() supported = dialect.supports_data_types() @@ -334,26 +371,32 @@ def test_returns_dict(self): suggested = dialect.suggested_data_types() assert isinstance(suggested, dict) - def test_suggests_string_for_binary_types(self): + def test_suggests_string_for_byte_and_text_concepts(self): dialect = ClickHouseDialect() suggested = dialect.suggested_data_types() - for key in ("binary", "varbinary", "blob"): - if key in suggested: - assert suggested[key] is ClickHouseStringType, ( - f"suggested type for {key!r} should be ClickHouseStringType" - ) + for key in ("binary", "varbinary", "custom", "enum", "jsonb"): + assert suggested[key] is ClickHouseStringType, key - def test_suggests_string_for_enum(self): + def test_suggests_what_clickhouse_really_stores_for_each(self): + """An interval is a count of seconds; a timezone-aware instant is a + ``DateTime64``; a time of day has no ClickHouse type at all; XML is not in + the manual's list of types.""" dialect = ClickHouseDialect() suggested = dialect.suggested_data_types() - if "enum" in suggested: - assert suggested["enum"] is ClickHouseStringType + assert suggested["interval"] is ClickHouseInt64Type + assert suggested["timestamptz"] is ClickHouseDateTime64Type + assert suggested["timetz"] is ClickHouseDateTimeType + assert suggested["xml"] is TextType - def test_suggests_string_for_jsonb(self): + def test_every_entry_has_a_reason(self): + """A suggestion with no explanation is a guess the reader has to check.""" dialect = ClickHouseDialect() - suggested = dialect.suggested_data_types() - if "jsonb" in suggested: - assert suggested["jsonb"] is ClickHouseStringType + doc = (type(dialect).suggested_data_types.__doc__ or "") + for concept in dialect.suggested_data_types(): + assert concept in doc, ( + f"suggested_data_types() offers {concept!r} but its docstring " + f"never says why" + ) def test_no_overlap_with_supported(self): dialect = ClickHouseDialect() @@ -410,17 +453,713 @@ def test_decimal128_scale_validation(self): # ── W4: Float has no precision constraint ───────────────────────────── class TestFloatNoPrecision: + """``Float32``/``Float64`` carry no adjustable precision. + + ClickHouse's float types are fixed widths — there is no ``FLOAT(p)`` spelling + to widen the mantissa to — so neither class has a ``precision`` parameter, and + ``PARAMETERS`` is stated rather than inherited so that equality does not + report a parameter the constructor will not accept. + """ + def test_float32_no_params(self): + # Declared empty on purpose: ``__init__`` takes no precision, so + # inheriting the base's ``precision`` would advertise an identity the + # class cannot have. + assert ClickHouseFloat32Type.PARAMETERS == () t = ClickHouseFloat32Type() - assert t._type_params() == () + assert t.identity() == () + assert t.get_params() == {} def test_float64_no_params(self): + assert ClickHouseFloat64Type.PARAMETERS == () t = ClickHouseFloat64Type() - assert t._type_params() == () + assert t.identity() == () + assert t.get_params() == {} + + @pytest.mark.parametrize("cls", [ClickHouseFloat32Type, ClickHouseFloat64Type]) + def test_float_rejects_precision(self, cls): + """A precision the server cannot honour must not be accepted silently.""" + with pytest.raises(TypeError, match="precision"): + cls(precision=5) + + @pytest.mark.parametrize("cls", [ClickHouseFloat32Type, ClickHouseFloat64Type]) + def test_float_rejects_a_second_positional_argument(self, cls): + """``dialect`` is first: a stray positional must not land in ``precision``.""" + with pytest.raises(TypeError): + cls(None, 5) + + +class TestDateTimeTypeHasNoPrecision: + """``DateTime`` is whole seconds; ``DateTime64(p)`` is the variable one. + + Same reasoning as the floats, and the same risk in the other direction: + :class:`TimestampType` reads ``precision``, so an inherited constructor would + accept a sub-second precision that ``DateTime`` cannot store. + """ + + def test_datetime_type_is_hashable(self): + t = ClickHouseDateTimeType() + assert t == ClickHouseDateTimeType() + assert hash(t) == hash(ClickHouseDateTimeType()) + assert isinstance(t.get_params(), dict) + + def test_datetime_type_has_no_params(self): + assert ClickHouseDateTimeType.PARAMETERS == () + t = ClickHouseDateTimeType() + assert t.identity() == () + assert t.get_params() == {} + + def test_datetime_type_rejects_precision(self): + with pytest.raises(TypeError, match="precision"): + ClickHouseDateTimeType(precision=3) + + def test_datetime64_keeps_precision(self): + assert ClickHouseDateTime64Type.PARAMETERS == ("precision",) + assert ClickHouseDateTime64Type(precision=6).get_params() == {"precision": 6} + assert ClickHouseDateTime64Type(precision=6).identity() == (6,) + assert ClickHouseDateTime64Type(precision=6) != ClickHouseDateTime64Type() + + +# ── D1: every ClickHouse type that is a core concept derives from it ── + +#: ClickHouse type -> the core concept it is. Asserted rather than derived from +#: ``__mro__`` because that is what the refactor changed: a ClickHouse type left +#: on ``DataType`` would still render the same SQL and still pass every dispatch +#: test, while ``isinstance`` would answer "no" to "is this an 8-byte integer?" — +#: which is the question the hierarchy exists to answer once, not per backend. +REPARENTED_TO_CORE_CONCEPT = { + ClickHouseInt8Type: TinyIntType, + ClickHouseUInt8Type: TinyIntType, + ClickHouseInt16Type: SmallIntType, + ClickHouseUInt16Type: SmallIntType, + ClickHouseInt32Type: IntegerType, + ClickHouseUInt32Type: IntegerType, + ClickHouseInt64Type: BigIntType, + ClickHouseUInt64Type: BigIntType, + ClickHouseFloat32Type: FloatType, + ClickHouseFloat64Type: DoubleType, + ClickHouseDecimalType: DecimalType, + ClickHouseDecimal32Type: DecimalType, + ClickHouseDecimal64Type: DecimalType, + ClickHouseDecimal128Type: DecimalType, + ClickHouseStringType: TextType, + ClickHouseFixedStringType: BinaryType, + ClickHouseDateType: DateType, + ClickHouseDate32Type: DateType, + ClickHouseDateTimeType: TimestampType, + ClickHouseDateTime64Type: TimestampType, + ClickHouseBoolType: BooleanType, + ClickHouseUUIDType: UUIDType, + ClickHouseJSONType: JsonType, +} + + +class TestCoreConceptReParenting: + @pytest.mark.parametrize("cls,base", sorted( + REPARENTED_TO_CORE_CONCEPT.items(), + key=lambda pair: pair[0].__name__)) + def test_direct_base_is_the_core_concept(self, cls, base): + assert cls.__bases__ == (base,), ( + f"{cls.__name__} should derive directly from {base.__name__}, " + f"not {', '.join(b.__name__ for b in cls.__bases__)}" + ) + + @pytest.mark.parametrize( + "cls", sorted(REPARENTED_TO_CORE_CONCEPT, key=lambda c: c.__name__)) + def test_reparenting_did_not_change_the_name(self, cls): + """``name`` is the dispatch key; re-parenting must not move it. + + ``ClickHouseInt32Type`` deriving from ``IntegerType`` does **not** make + its name ``integer``: the DDL says ``Int32`` and a schema diff compares + the word the server wrote. + """ + assert cls.name.startswith("clickhouse_") + + def test_every_clickhouse_type_is_either_reparented_or_on_the_root_list(self): + """Otherwise a class was added and neither list was updated. + + ``ALL_CLICKHOUSE_TYPES`` is the canonical inventory; the check is that the + two lists between them account for it exactly. + """ + accounted = set(REPARENTED_TO_CORE_CONCEPT) | set(ON_DATA_TYPE) \ + | set(ON_A_DOCUMENTED_BACKEND_BASE) | {ClickHouseArrayType} + missing = sorted(cls.__name__ for cls in ALL_CLICKHOUSE_TYPES + if cls not in accounted) + assert not missing, missing + + def test_unsigned_clickhouse_integers_say_so(self): + """D5: signedness is a field, and here it is not left lying.""" + for signed, unsigned in ( + (ClickHouseInt8Type, ClickHouseUInt8Type), + (ClickHouseInt16Type, ClickHouseUInt16Type), + (ClickHouseInt32Type, ClickHouseUInt32Type), + (ClickHouseInt64Type, ClickHouseUInt64Type), + ): + assert signed().unsigned is False + assert unsigned().unsigned is True + + def test_fixed_string_is_the_byte_string_concept(self): + """``FixedString(N)`` is N **bytes**, zero-padded — not ``CHAR(N)``. + + The padding is the whole argument: a fixed-length *character* type cannot + make ``FixedString(3)`` holding ``'a'`` equal ``'a'`` but not match + ``LIKE 'a'``, which is exactly what zero-filling produces. + """ + assert issubclass(ClickHouseFixedStringType, BinaryType) + assert not issubclass(ClickHouseFixedStringType, CharType) + assert not issubclass(ClickHouseFixedStringType, TextType) + + def test_datetime_is_a_timestamp_not_a_mysql_datetime(self): + """ClickHouse's ``DateTime`` is a count of seconds since the epoch. + + That is SQL's ``TIMESTAMP``. MySQL's ``DATETIME`` has no epoch to convert + against, and the two are not interchangeable, so the parent is the one + whose storage is an instant. + """ + from rhosocial.activerecord.backend.expression.types import DateTimeType + assert issubclass(ClickHouseDateTimeType, TimestampType) + assert not issubclass(ClickHouseDateTimeType, DateTimeType) + + +# ── D1/D4: the chain is linear — no diamonds, no grouping nodes ────── + +def _all_data_type_subclasses(): + seen, stack, out = set(), [DataType], [] + while stack: + klass = stack.pop() + if klass in seen: + continue + seen.add(klass) + stack.extend(klass.__subclasses__()) + if klass is not DataType: + out.append(klass) + return out + + +_CORE_MODULE = "rhosocial.activerecord.backend.expression.types" + + +def _core_concepts(): + """The concrete concepts core itself owns, keyed by dispatch ``name``. + + Discovered rather than listed so that a concept added to core later is + covered here without an edit — which is the whole point of the + expression/protocol correspondence: the set of concepts is open. + """ + return { + klass.name: klass + for klass in _all_data_type_subclasses() + if getattr(klass, "__module__", "").startswith(_CORE_MODULE) + and getattr(klass, "name", None) + } + + +def _core_concept_by_name(): + return _core_concepts() + + +class TestChainIsLinear: + def test_no_multiple_inheritance_among_data_type_subclasses(self): + """D1/D4: width is a class and signedness a field, so no (width x sign) + grid — which means no class can have two bases.""" + offenders = [ + klass.__name__ for klass in _all_data_type_subclasses() + if len(klass.__bases__) > 1 + ] + assert not offenders, offenders + + def test_every_backend_type_base_is_a_data_type(self): + for klass in _all_data_type_subclasses(): + for base in klass.__bases__: + assert issubclass(base, DataType), ( + f"{klass.__name__} derives from {base.__name__}, which is " + f"not a DataType" + ) + + +class TestDialectFirstConstructor: + """``dialect`` is the first positional parameter of every ``DataType``. + + A signature that puts a business parameter first fails *silently*: the dialect + object is stored as the bit count and the error only appears at render time. + """ + + def test_dialect_is_first_everywhere(self): + offenders = [] + for klass in _all_data_type_subclasses(): + params = list(inspect.signature(klass.__init__).parameters.values()) + if not params or params[0].name == "self": + continue + first = params[1] if params[0].name == "self" else params[0] + if first.name != "dialect": + offenders.append(f"{klass.__name__}({first.name}, ...)") + assert not offenders, offenders + + def test_dialect_cannot_be_stored_as_a_length_or_a_count(self): + """The failure this convention prevents, asserted where it would bite.""" + sentinel = object() + with pytest.raises(TypeError): + ClickHouseFixedStringType(sentinel, 8) + with pytest.raises(TypeError): + ClickHouseEnum8Type(sentinel, [("a", 1)]) + with pytest.raises(TypeError): + ClickHouseVectorType(sentinel, 4) + + +# ── D7: the classes that stay on DataType say why ──────────────────── + +#: ClickHouse types with no core concept to be, that sit **directly** on +#: ``DataType``. Each must argue the point in its docstring — see +#: :class:`TestRootSittingTypesAreDocumented`. +ON_DATA_TYPE = [ + ClickHouseEnum8Type, ClickHouseEnum16Type, + ClickHouseNullableType, ClickHouseLowCardinalityType, + ClickHouseMapType, ClickHouseTupleType, + ClickHouseAggregateFunctionType, ClickHouseSimpleAggregateFunctionType, + ClickHouseIPv4Type, ClickHouseIPv6Type, + ClickHouseGeometryType, ClickHouseVectorType, +] + +#: ClickHouse types with no core concept that are *not* on ``DataType`` either, +#: because they are correct: ``ClickHouseArrayType`` is core's ``ArrayType`` in +#: ClickHouse's spelling, and the spatial classes hang off +#: :class:`ClickHouseGeometryType`, which is ClickHouse's own real type as well +#: as their base — not a grouping node invented to hold them. +ON_A_DOCUMENTED_BACKEND_BASE = [ + ClickHousePointType, ClickHouseLineStringType, ClickHousePolygonType, + ClickHouseMultiPointType, ClickHouseMultiLineStringType, + ClickHouseMultiPolygonType, ClickHouseGeometryCollectionType, +] + + +class TestRootSittingTypesAreDocumented: + """D7: sitting on ``DataType`` is legitimate but must be *argued*. + + Not an exemption with a marker — there is no allowlist, no annotation and no + registry anywhere — so the requirement is the docstring itself: it has to say + why this is not a core concept, with the concrete difference, not just name + the type. + """ + + @pytest.mark.parametrize("cls", ON_DATA_TYPE, ids=lambda c: c.__name__) + def test_sits_directly_on_data_type(self, cls): + assert cls.__bases__ == (DataType,), ( + f"{cls.__name__} is listed as a ClickHouse-only type but derives " + f"from {', '.join(b.__name__ for b in cls.__bases__)}" + ) + + @pytest.mark.parametrize("cls", ON_DATA_TYPE, ids=lambda c: c.__name__) + def test_docstring_explains_why_not_core(self, cls): + doc = (cls.__doc__ or "").strip() + assert len(doc) > 120, ( + f"{cls.__name__}.__doc__ is {len(doc)} chars — D7 wants the reason " + f"this is not a core concept, not a restatement of the type name" + ) + lowered = doc.lower() + assert "core" in lowered or "sql:2016" in lowered, ( + f"{cls.__name__}.__doc__ never mentions core or SQL:2016, so it " + f"cannot be saying why it is not a core concept" + ) + + @pytest.mark.parametrize( + "cls", ON_A_DOCUMENTED_BACKEND_BASE, ids=lambda c: c.__name__) + def test_backend_internal_base_is_a_real_type_not_a_grouping_node(self, cls): + """The seven spatial classes share a base, and D1 forbids a group node. + + The escape is that the base is not invented: ClickHouse *names* + ``GEOMETRY`` as a type in its own right and a column declared ``GEOMETRY`` + accepts any of the seven. Asserted so the arrangement cannot be copied + into a real grouping node later. + """ + assert issubclass(cls, ClickHouseGeometryType), ( + f"{cls.__name__} does not hang off ClickHouseGeometryType" + ) + base = cls.__bases__[0] + assert base.__module__.startswith(_CORE_MODULE) \ + or ".impl.clickhouse." in base.__module__, ( + f"{cls.__name__} derives from {base.__name__}, which is neither a " + f"core concept nor a ClickHouse type" + ) + assert (cls.__doc__ or "").strip(), f"{cls.__name__} has no docstring" + + def test_array_is_core_in_clickhouse_spelling(self): + """``ClickHouseArrayType`` is not in either "not a core concept" list + because it *is* one — core's ``ArrayType``, written the way ClickHouse + writes it.""" + assert ClickHouseArrayType.__bases__ == (ArrayType,) + assert ClickHouseArrayType.name == "clickhouse_array" + + def test_every_non_core_clickhouse_type_is_on_one_of_the_two_lists(self): + """Neither list may silently grow a class the other does not know about.""" + known = set(REPARENTED_TO_CORE_CONCEPT) | set(ON_DATA_TYPE) \ + | set(ON_A_DOCUMENTED_BACKEND_BASE) | {ClickHouseArrayType} + unknown = sorted(cls.__name__ for cls in ALL_CLICKHOUSE_TYPES + if cls not in known) + assert not unknown, unknown + + def test_enum_docstring_lists_the_consequences(self): + """Not "this is Enum8" — the *consequences* of it not being ``EnumType``. + + Checked against ClickHouse's own documentation: explicit possibly-negative + integers, ordering by the assigned integer, non-nullable, and undefined + for most numeric and string operations. + """ + for cls in (ClickHouseEnum8Type, ClickHouseEnum16Type): + doc = (cls.__doc__ or "").lower() + assert "integer" in doc, cls.__name__ + assert "order" in doc, cls.__name__ + assert "null" in doc, cls.__name__ + assert "undefined" in doc, cls.__name__ + + def test_low_cardinality_is_a_storage_encoding(self): + """It is not a different type from its inner type, which is the reason it + cannot derive from one.""" + doc = (ClickHouseLowCardinalityType.__doc__ or "").lower() + assert "encoding" in doc + assert not issubclass(ClickHouseLowCardinalityType, ClickHouseStringType) + + +# ── D9: expression <-> protocol correspondence ──────────────────────── + +class TestCorrespondence: + """Every concrete core concept is rendered or named as a substitute.""" + + def test_every_core_concept_is_declared(self): + dialect = ClickHouseDialect() + supported = dialect.supports_data_types() + suggested = dialect.suggested_data_types() + undeclared = sorted( + name for name in _core_concepts() + if name not in supported and name not in suggested + ) + assert not undeclared, undeclared + + def test_supported_and_suggested_are_disjoint(self): + dialect = ClickHouseDialect() + overlap = set(dialect.supports_data_types()) & set(dialect.suggested_data_types()) + assert not overlap, overlap + + def test_every_suggested_substitute_is_renderable(self): + """A substitute the dialect cannot render is advice nobody can take.""" + dialect = ClickHouseDialect() + supported = dialect.supports_data_types() + for concept, substitute in dialect.suggested_data_types().items(): + assert isinstance(substitute, type) and issubclass(substitute, DataType), ( + f"{concept} -> {substitute!r} is not a DataType subclass" + ) + assert substitute.name in supported, ( + f"{concept} -> {substitute.__name__} (name={substitute.name!r}), " + f"which this dialect does not render" + ) + + def test_concepts_clickhouse_cannot_spell_are_all_substituted(self): + dialect = ClickHouseDialect() + suggested = dialect.suggested_data_types() + # ClickHouse's manual enumerates its types; these are the concepts that + # are genuinely not among them. + for concept in ("custom", "interval", "timestamptz", "timetz", "xml"): + assert concept in suggested, f"{concept} is undeclared" + assert concept not in dialect.supports_data_types(), concept + + def test_uuid_is_rendered_not_substituted(self): + """ClickHouse has a native ``UUID``, so the concept is rendered.""" + dialect = ClickHouseDialect() + assert "uuid" in dialect.supports_data_types() + assert "uuid" not in dialect.suggested_data_types() + assert dialect.format_data_type(UUIDType(dialect))[0] == "UUID" + + def test_error_for_an_unrendered_concept_names_the_substitute(self): + """Silence is the one forbidden answer: the message must say what to use.""" + from rhosocial.activerecord.backend.expression.types import IntervalType + dialect = ClickHouseDialect() + with pytest.raises(TypeError) as excinfo: + dialect.format_data_type(IntervalType(dialect)) + assert "ClickHouseInt64Type" in str(excinfo.value) + + def test_backend_type_protocol_is_in_the_mro_before_the_generic_one(self): + mro = ClickHouseDialect.__mro__ + assert ClickHouseTypeSupport in mro + assert mro.index(ClickHouseTypeSupport) < mro.index(DataTypeSupport) + assert isinstance(ClickHouseDialect(), ClickHouseTypeSupport) + + def test_protocol_does_not_absorb_the_set_type_contract(self): + """``ClickHouseSetTypeSupport`` is about functions, not a column type.""" + from rhosocial.activerecord.backend.impl.clickhouse.protocols import ( + ClickHouseSetTypeSupport, + ) + assert ClickHouseSetTypeSupport not in ClickHouseTypeSupport.__mro__ + + +class TestSpellingGates: + """D9: a formatter states which spellings it renders and refuses the rest.""" + + SPELLING_RENDERINGS = { + "integer": {"integer": "Int32", "int": "Int32"}, + "bigint": {"bigint": "Int64", "int8": "Int64"}, + "smallint": {"smallint": "Int16", "int2": "Int16"}, + "tinyint": {"tinyint": "Int8", "int1": "Int8"}, + "char": {"char": "String", "character": "String"}, + "varchar": {"varchar": "String", "character varying": "String"}, + "text": {"text": "String", "clob": "String"}, + "boolean": {"boolean": "Bool", "bool": "Bool"}, + "decimal": { + "decimal": "Decimal(10, 0)", + "numeric": "Decimal(10, 0)", + "dec": "Decimal(10, 0)", + }, + "double": {"double": "Float64", "double precision": "Float64"}, + "blob": {"blob": "String", "bytea": "String"}, + } + + @pytest.mark.parametrize("concept", sorted(SPELLING_RENDERINGS)) + def test_every_declared_spelling_renders(self, concept): + dialect = ClickHouseDialect() + cls = _core_concept_by_name()[concept] + for spelling, expected in self.SPELLING_RENDERINGS[concept].items(): + sql, _ = dialect.format_data_type(cls(dialect, spelling=spelling)) + assert sql == expected, f"{concept}[{spelling!r}] -> {sql}" + + def test_default_spelling_always_renders(self): + """The core invariant: a concept must be usable on this backend.""" + dialect = ClickHouseDialect() + for name, cls in sorted(_core_concept_by_name().items()): + if not getattr(cls, "SPELLINGS", ()): + continue + sql, _ = dialect.format_data_type(cls(dialect)) + assert sql, name + + @pytest.mark.parametrize("concept", sorted(SPELLING_RENDERINGS)) + def test_a_spelling_outside_the_list_is_refused_by_name(self, concept): + """No silent fallback: the error names the spelling that was rejected.""" + dialect = ClickHouseDialect() + data_type = _core_concept_by_name()[concept]( + dialect, spelling="not_a_real_spelling") + with pytest.raises(TypeError, match="not_a_real_spelling"): + dialect.format_data_type(data_type) + + def test_clickhouse_spells_all_of_a_concepts_spellings(self): + """``accepted`` may be an explicit tuple for a backend that spells only + some of a concept. ClickHouse spells all of them, so the concept class is + passed to ``_check_spelling`` — asserted here so narrowing any one of them + has to be deliberate rather than a copy-paste slip.""" + dialect = ClickHouseDialect() + # ``clob`` is not a ClickHouse word, yet it is accepted and rendered + # ``String``; refusing it would refuse the concept's own second spelling + # without saying anything the rendered column does not say. + assert dialect.format_data_type(TextType(dialect, spelling="clob"))[0] == "String" + + +# ── D8: parse_type is canonical ────────────────────────────────────── + +class TestParseTypeCanonicality: + @pytest.fixture + def dialect(self): + return ClickHouseDialect() + + @pytest.mark.parametrize("raw,expected", [ + ("Decimal32(4)", "Decimal32(4)"), + ("Decimal64(8)", "Decimal64(8)"), + ("Decimal128(18)", "Decimal128(18)"), + ("Decimal32", "Decimal32(0)"), + ("Decimal(10, 2)", "Decimal(10, 2)"), + ("Decimal(10)", "Decimal(10, 0)"), + ]) + def test_decimal_width_digits_are_not_mistaken_for_the_scale(self, dialect, raw, expected): + """The digits in the *name* are the width. + + ``Decimal32(4)`` used to parse as ``Decimal32(32)``, and + ``Decimal64(8)`` / ``Decimal128(18)`` raised ``ValueError`` because the + width came back as a scale outside 0..38. + """ + assert dialect.format_data_type(dialect.parse_type(raw))[0] == expected + + @pytest.mark.parametrize("raw", ["Bool", "BOOLEAN"]) + def test_boolean_spellings_are_one_class(self, dialect, raw): + """ClickHouse accepts ``Bool`` and ``BOOLEAN`` for the same type.""" + parsed = dialect.parse_type(raw) + assert isinstance(parsed, ClickHouseBoolType) + assert dialect.format_data_type(parsed)[0] == "Bool" + + @pytest.mark.parametrize("raw", ["Decimal(10, 2)", "NUMERIC(10, 2)", "DEC(10, 2)"]) + def test_decimal_spellings_are_one_class(self, dialect, raw): + """The core ``SPELLINGS`` list is exactly ClickHouse's own vocabulary.""" + parsed = dialect.parse_type(raw) + assert isinstance(parsed, ClickHouseDecimalType) + assert dialect.format_data_type(parsed)[0] == "Decimal(10, 2)" + + def test_enum_order_is_canonicalised_by_the_assigned_integer(self, dialect): + """ClickHouse reorders enum labels by their integer, not by declaration. + + A live 26.7 server reports ``Enum8('x' = 1, 'y' = -2)`` back as + ``Enum8('y' = -2, 'x' = 1)``, so a parser that kept the written order + would make an introspected column differ from the declaration that + produced it. + """ + assert dialect.parse_type("Enum8('x' = 1, 'y' = -2)") == \ + dialect.parse_type("Enum8('y' = -2, 'x' = 1)") + + @pytest.mark.parametrize("raw", [ + "Int8", "UInt64", "Float32", "Decimal32(4)", "String", "FixedString(8)", + "Date", "Date32", "DateTime", "DateTime64(3)", "Bool", "UUID", "IPv4", + "IPv6", "JSON", "Array(Int32)", "Map(String, Int32)", + "Tuple(String, Int32)", "Nullable(Int32)", "LowCardinality(String)", + "Enum8('a' = 1)", + ]) + def test_parse_then_render_is_a_fixed_point(self, dialect, raw): + parsed = dialect.parse_type(raw) + assert dialect.format_data_type(parsed)[0] == raw + + def test_an_unknown_name_is_not_silently_written_out(self, dialect): + """ClickHouse's type grammar is closed, so there is no + ``format_data_type_custom`` — the fallback must be a *render failure* + with advice, not a pass-through.""" + from rhosocial.activerecord.backend.expression.types import CustomType + parsed = dialect.parse_type("SomeTypeWeDoNotKnow") + assert isinstance(parsed, CustomType) + with pytest.raises(TypeError) as excinfo: + dialect.format_data_type(parsed) + assert "ClickHouseStringType" in str(excinfo.value) + + def test_sql_standard_words_are_not_claimed_as_clickhouse_types(self, dialect): + """``VARCHAR(10)`` is not claimed; the server's alias table is wider. + + Measured on 26.7.3.19: ``CREATE TABLE t (a VARCHAR(10))`` is actually + accepted and resolves to ``String`` — it is one of the server's 73 + aliases — but this parser claims the native names this dialect renders, + not the whole alias table. An unclaimed word falls through to + ``CustomType``, which cannot be rendered, so the caller gets a reported + failure rather than a wrong declaration. + """ + from rhosocial.activerecord.backend.expression.types import CustomType + for raw in ("VARCHAR(10)", "TINYINT", "BYTEA", "CLOB"): + try: + parsed = dialect.parse_type(raw) + except Exception: + continue + assert isinstance(parsed, CustomType), raw + + @pytest.mark.parametrize("raw", [ + "TINYINT", "INT1", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT", + ]) + def test_the_single_word_signed_aliases_are_not_claimed(self, dialect, raw): + """The other half of the alias table is deliberately left where it was. + + The server accepts every one of these and resolves them to the *signed* + widths — ``toTypeName(CAST(1 AS TINYINT))`` is ``Int8`` on 26.7.3.19 — + but they are alternate spellings of the signed columns whose native names + (``Int8``…``Int64``) this parser already claims, and the server's alias + table is wider than the vocabulary this dialect reads (``FLOAT``, + ``DOUBLE`` and ``VARCHAR(10)`` are unclaimed too). A single word falls + through to ``CustomType``, exactly as before the ``* UNSIGNED`` family + was recognised: "not claimed", not "rejected by the server". + """ + from rhosocial.activerecord.backend.expression.types import CustomType + assert isinstance(dialect.parse_type(raw), CustomType), raw + + def test_case_is_tolerated_even_though_native_type_names_are_case_sensitive( + self, dialect): + """Documented leniency, asserted so it stays a decision. + + Case decides only the *native* names, and only at the server: measured + on 26.7.3.19, ``CREATE TABLE t (a Int8)`` and ``(a String)`` succeed + while ``(a INT8)``, ``(a int8)`` and ``(a string)`` come back code 50 + ``Unknown data type family`` (the alias words — ``TINYINT``, + ``VARCHAR``, ``YEAR`` and friends — are ``case_insensitive = 1``). + ``parse_type`` upper-cases everything, so it tolerates any case — + harmless for its introspection caller, which is handed the server's own + canonical output, and convenient for hand-written DDL. + """ + for raw, expected in ( + ("int8", "Int8"), + ("STRING", "String"), + ("boolean", "Bool"), + ): + parsed = dialect.parse_type(raw) + assert dialect.format_data_type(parsed)[0] == expected, raw # ── format_data_type dispatch ───────────────────────────────────────── +#: ``{label: rendered SQL}`` captured from the tree *before* the re-parenting. +#: +#: The point of re-parenting is that it is a pure structural change: a ClickHouse +#: type that becomes a subclass of a core concept must render byte-for-byte what +#: it rendered when it sat on ``DataType``. A golden table is the only evidence +#: that does not depend on comparing two checkouts at test time, and a subclass +#: can silently start rendering differently (through an inherited declaration, +#: a changed default, a lost width) while every dispatch test still passes. +BEFORE_REPARENTING_SQL = { + "ClickHouseInt8Type": "Int8", + "ClickHouseInt16Type": "Int16", + "ClickHouseInt32Type": "Int32", + "ClickHouseInt64Type": "Int64", + "ClickHouseUInt8Type": "UInt8", + "ClickHouseUInt16Type": "UInt16", + "ClickHouseUInt32Type": "UInt32", + "ClickHouseUInt64Type": "UInt64", + "ClickHouseFloat32Type": "Float32", + "ClickHouseFloat64Type": "Float64", + "ClickHouseDecimalType": "Decimal(18, 4)", + "ClickHouseDecimal32Type": "Decimal32(4)", + "ClickHouseDecimal64Type": "Decimal64(8)", + "ClickHouseDecimal128Type": "Decimal128(18)", + "ClickHouseStringType": "String", + "ClickHouseFixedStringType": "FixedString(16)", + "ClickHouseDateType": "Date", + "ClickHouseDate32Type": "Date32", + "ClickHouseDateTimeType": "DateTime", + "ClickHouseDateTime64Type": "DateTime64(3)", + "ClickHouseBoolType": "Bool", + "ClickHouseUUIDType": "UUID", + "ClickHouseJSONType": "JSON", + "ClickHouseIPv4Type": "IPv4", + "ClickHouseIPv6Type": "IPv6", + "ClickHouseGeometryType": "GEOMETRY", + "ClickHousePointType": "POINT", + "ClickHouseLineStringType": "LINESTRING", + "ClickHousePolygonType": "POLYGON", + "ClickHouseMultiPointType": "MULTIPOINT", + "ClickHouseMultiLineStringType": "MULTILINESTRING", + "ClickHouseMultiPolygonType": "MULTIPOLYGON", + "ClickHouseGeometryCollectionType": "GEOMETRYCOLLECTION", +} + + +class TestRenderingIsUnchangedByReParenting: + """The golden table above, asserted against today's rendering.""" + + ARGS = { + "ClickHouseDecimalType": {"precision": 18, "scale": 4}, + "ClickHouseDecimal32Type": {"scale": 4}, + "ClickHouseDecimal64Type": {"scale": 8}, + "ClickHouseDecimal128Type": {"scale": 18}, + "ClickHouseFixedStringType": {"length": 16}, + "ClickHouseDateTime64Type": {"precision": 3}, + } + + @pytest.mark.parametrize("cls_name", sorted(BEFORE_REPARENTING_SQL)) + def test_rendered_sql_matches_the_pre_reparenting_snapshot(self, cls_name): + dialect = ClickHouseDialect() + cls = _all_data_type_subclasses_by_name()[cls_name] + sql, params = cls(dialect, **self.ARGS.get(cls_name, {})).to_sql() + assert sql == BEFORE_REPARENTING_SQL[cls_name] + assert params == () + + @pytest.mark.parametrize("cls_name", sorted(BEFORE_REPARENTING_SQL)) + def test_the_golden_table_covers_every_reparented_class(self, cls_name): + """A class re-parented without being added here would go unchecked. + + The reverse direction is not asserted: the table also covers the classes + that legitimately stay on ``DataType`` (the spatial family), because those + render exactly as they did before the refactor too. + """ + assert cls_name in {c.__name__ for c in ALL_CLICKHOUSE_TYPES}, cls_name + + +def _all_data_type_subclasses_by_name(): + return {klass.__name__: klass for klass in _all_data_type_subclasses()} + + class TestFormatDispatch: def test_format_clickhouse_int8(self): dialect = ClickHouseDialect() @@ -440,19 +1179,880 @@ def test_format_clickhouse_enum8(self): assert sql == "Enum8('active' = 1, 'inactive' = 0)" def test_format_core_integer_maps_to_int32(self): - from rhosocial.activerecord.backend.expression.types import IntegerType dialect = ClickHouseDialect() sql, _ = IntegerType(dialect).to_sql() assert sql == "Int32" def test_format_core_varchar_maps_to_string(self): - from rhosocial.activerecord.backend.expression.types import VarCharType dialect = ClickHouseDialect() sql, _ = VarCharType(dialect).to_sql() assert sql == "String" def test_format_unsupported_type_raises(self): + """A concept with no renderer and no substitute must still fail loudly. + + ``uuid`` used to be the example here, and no longer is: ClickHouse has a + native ``UUID``, so the concept is rendered (see + :meth:`TestCorrespondence.test_uuid_is_rendered_not_substituted`). What is + left genuinely unrenderable on this backend is an unrecognised type name, + because ClickHouse's type grammar is closed and there is no + ``format_data_type_custom`` to pass it through. + """ + from rhosocial.activerecord.backend.expression.types import CustomType + dialect = ClickHouseDialect() + with pytest.raises(TypeError, match="does not support the generic type 'custom'"): + CustomType(dialect, "SomeTypeWeDoNotKnow").to_sql() + + def test_format_unknown_type_error_names_the_substitute(self): + """The message has to say what to use, not just that nothing is there.""" + from rhosocial.activerecord.backend.expression.types import CustomType + dialect = ClickHouseDialect() + with pytest.raises(TypeError, match="ClickHouseStringType"): + CustomType(dialect, "SomeTypeWeDoNotKnow").to_sql() + + +# ── ``unsigned`` is refused, not dropped ───────────────────────────── +# +# ``unsigned`` is in the four core integer concepts' ``PARAMETERS``, so it is part +# of their identity: the differ compares ``ClickHouseInt8Type(unsigned=True)`` +# against ``ClickHouseInt8Type()`` and sees two different columns. A dialect must +# therefore honour it or refuse it. ClickHouse refuses, because it spells +# signedness in the type *name* — ``Int8`` and ``UInt8`` are two types over one +# storage size, and ``UNSIGNED`` is not an attribute a declaration can carry: +# ``CREATE TABLE t (a Int8 UNSIGNED)`` fails with ``Unknown data type family: +# INT8 UNSIGNED`` on 26.7.3.19. +# +# https://clickhouse.com/docs/sql-reference/data-types/int-uint +# +# Honouring it was available and was still refused: ClickHouse does accept the +# MySQL-compat alias ``TINYINT UNSIGNED`` (``SELECT toTypeName(CAST(1 AS +# TINYINT UNSIGNED))`` returns ``UInt8``), but that column is byte-for-byte the one +# ``ClickHouseUInt8Type`` already renders. Two declarations for one column is what +# one-concept-one-class forbids. + +#: (backend class, core class reached by the same name, signed word, unsigned word, +#: unsigned class) +UNSIGNED_PAIRS = [ + (ClickHouseInt8Type, TinyIntType, "Int8", "UInt8", "ClickHouseUInt8Type"), + (ClickHouseInt16Type, SmallIntType, "Int16", "UInt16", "ClickHouseUInt16Type"), + (ClickHouseInt32Type, IntegerType, "Int32", "UInt32", "ClickHouseUInt32Type"), + (ClickHouseInt64Type, BigIntType, "Int64", "UInt64", "ClickHouseUInt64Type"), +] + + +class TestUnsignedIsRefusedNotDropped: + @pytest.mark.parametrize( + "signed, core, signed_word, unsigned_word, unsigned_class", + UNSIGNED_PAIRS, ids=[pair[2] for pair in UNSIGNED_PAIRS]) + def test_signed_rendering_is_unchanged( + self, signed, core, signed_word, unsigned_word, unsigned_class): + """The default declaration still renders exactly what it always did.""" + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert signed(dialect).to_sql()[0] == signed_word + assert dialect.format_data_type(core(dialect))[0] == signed_word + + @pytest.mark.parametrize( + "signed, core, signed_word, unsigned_word, unsigned_class", + UNSIGNED_PAIRS, ids=[pair[2] for pair in UNSIGNED_PAIRS]) + def test_unsigned_on_the_signed_class_raises( + self, signed, core, signed_word, unsigned_word, unsigned_class): + """Both dispatch keys refuse, because both can receive the flag. + + ``clickhouse_int8`` and ``tinyint`` are different dispatch keys for the + same concept, and ``unsigned`` is a field on the concept — so a caller who + reaches it through ``TinyIntType(unsigned=True)`` must be refused just as + loudly as one who reaches it through ``ClickHouseInt8Type(unsigned=True)``. + """ + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + dialect = ClickHouseDialect(version=(26, 7, 0)) + for declaration in (signed(dialect, unsigned=True), core(dialect, unsigned=True)): + with pytest.raises(UnsupportedFeatureError) as excinfo: + declaration.to_sql() + message = str(excinfo.value) + assert "unsigned" in message + assert signed_word in message, "the error must name the width" + assert unsigned_class in message, "the error must say what to use" + + def test_before_the_fix_the_sql_was_identical(self): + """What the defect was, stated as an executable claim. + + Two declarations the framework treats as two different columns — they are + ``!=`` because ``unsigned`` is in ``PARAMETERS`` — used to render the same + SQL. The only thing this backend may do about that is refuse one of them. + """ + dialect = ClickHouseDialect(version=(26, 7, 0)) + signed = ClickHouseInt8Type(dialect) + unsigned = ClickHouseInt8Type(dialect, unsigned=True) + assert signed != unsigned, "unsigned must still be part of the identity" + assert signed.unsigned is False and unsigned.unsigned is True + + @pytest.mark.parametrize( + "signed, core, signed_word, unsigned_word, unsigned_class", + UNSIGNED_PAIRS, ids=[pair[2] for pair in UNSIGNED_PAIRS]) + def test_the_unsigned_class_owns_the_unsigned_column( + self, signed, core, signed_word, unsigned_word, unsigned_class): + """One concept, one class: the unsigned column belongs to ``UInt*``. + + The class pins ``unsigned=True`` in its own ``__init__`` and takes no flag, + so there is nothing for its formatter to refuse and the ``supports_*`` + 1:1 pair keeps answering ``True``. + """ + dialect = ClickHouseDialect(version=(26, 7, 0)) + declaration = getattr(clickhouse_types, unsigned_class)(dialect) + assert declaration.unsigned is True + assert declaration.to_sql()[0] == unsigned_word + # dispatch is by name, so the 1:1 support method is the one named after it + assert getattr(dialect, f"supports_data_type_{declaration.name}")() is True + + def test_a_signed_and_an_unsigned_declaration_are_different_columns(self): + """The differ's input is different, so the renderer must not be silent.""" + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert ClickHouseInt8Type(dialect) != ClickHouseUInt8Type(dialect) + assert len({ClickHouseInt8Type(dialect), ClickHouseUInt8Type(dialect)}) == 2 + + +class TestUnsignedFloatAndDecimalAreRefused: + """ClickHouse's ``UInt8``…``UInt64`` are **integer families of their own**, and + ``Decimal`` / ``Float32`` / ``Float64`` have no unsigned counterpart at all — so + ``unsigned`` on those three is **refused**. + + Measured on the wired scenario server, 26.7.3.19: + + * ``system.data_type_families`` names 140 families. The eight whose names + contain ``UNSIGNED`` are all integer *aliases* and every one resolves to a + ``UInt`` family — ``TINYINT UNSIGNED``/``INT1 UNSIGNED`` → ``UInt8``, + ``SMALLINT UNSIGNED`` → ``UInt16``, + ``MEDIUMINT UNSIGNED``/``INT UNSIGNED``/``INTEGER UNSIGNED`` → ``UInt32``, + ``BIGINT UNSIGNED``/``UNSIGNED`` → ``UInt64``. There is **no** unsigned float or + decimal family among them, and the float and decimal families' own ``syntax`` + strings are the whole of their grammar: ``Float32``, ``Float64``, + ``Decimal(P, S)``, ``Decimal32(S)``…``Decimal256(S)``, ``BFloat16``. + * Nothing parses with the attribute appended. ``Decimal(10, 2) UNSIGNED``, + ``Decimal(10, 0) UNSIGNED``, ``Float32 UNSIGNED``, ``Float32(10) UNSIGNED`` and + ``Float64 UNSIGNED`` are all ``SYNTAX_ERROR`` code 62, + ``failed at position N (UNSIGNED)``. The expected-token list the server prints + back — ``COLLATE, NOT, NULL, DEFAULT, MATERIALIZED, ALIAS, EPHEMERAL, + AUTO_INCREMENT, COMMENT, CODEC, STATISTICS, TTL, PRIMARY KEY, SETTINGS, token, + Comma, ClosingRoundBracket`` — is everything that may follow a column type, and + no signedness attribute is on it. The **inline** ``CHECK (c >= 0)`` is rejected + by the same rule at the same position: the slot does not exist. + * Signedness in the name means the unsigned integers are a different kind of + thing, not a decorated one: ``CAST(-1 AS UInt8)`` is ``255``, while + ``CAST(-1 AS Float32)``, ``CAST(-1 AS Float64)`` and + ``CAST(-1 AS Decimal(10,2))`` are all ``-1``. So no reachable value differs, + and the only way to make one differ is to leave the float family. + + **The route forward is not the ``CHECK`` the other four backends suggest**, and + that difference is measured too: ``CREATE TABLE t (c Float32 CHECK (c >= 0))`` + fails with the same code-62 syntax error as ``UNSIGNED``, while + ``CREATE TABLE t (c Float32, CONSTRAINT cc CHECK (c >= 0))`` is accepted and + ``INSERT INTO t VALUES (-1)`` then fails with code 469, + ``Constraint 'cc' ... is violated ... Column values: c = -1``. ClickHouse does + enforce the constraint; it just has nowhere to put an inline one. + """ + + #: ``(label, class, builder, signed SQL, the ClickHouse word the refusal + #: names)``. + _FLOATS_AND_DECIMALS = [ + ("decimal", DecimalType, + lambda d, u: DecimalType(d, 10, 2, unsigned=u), + "Decimal(10, 2)", "Decimal(P, S)"), + ("float", FloatType, + lambda d, u: FloatType(d, unsigned=u), + "Float32", "Float32"), + ("double", DoubleType, + lambda d, u: DoubleType(d, unsigned=u), + "Float64", "Float64"), + ("real", RealType, + lambda d, u: RealType(d, unsigned=u), + "Float32", "Float32"), + ] + _IDS = [row[0] for row in _FLOATS_AND_DECIMALS] + + def _dialect(self): + return ClickHouseDialect(version=(26, 7, 0)) + + @pytest.mark.parametrize("label,klass,build,signed_sql,word", + _FLOATS_AND_DECIMALS, ids=_IDS) + def test_unsigned_is_refused_by_name(self, label, klass, build, signed_sql, + word): + """The rule: the field is in ``PARAMETERS``, so it must reach the SQL or + flip must raise. Byte-identical SQL for both signs is the violation — and + it is exactly what all four of these formatters used to produce.""" + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + dialect = self._dialect() + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.format_data_type(build(dialect, True)) + message = str(excinfo.value) + assert "unsigned" in message + assert word in message, "the error must name the type that has no variant" + assert "type NAME" in message, ( + "the error must say where ClickHouse puts signedness instead" + ) + + @pytest.mark.parametrize("label,klass,build,signed_sql,word", + _FLOATS_AND_DECIMALS, ids=_IDS) + def test_the_refusal_points_at_the_table_level_constraint( + self, label, klass, build, signed_sql, word): + """``CHECK`` is the route forward on four backends and is a **syntax error** + on this one; only the named table-level form is accepted, and the server + enforces it. Suggesting the inline form would be worse than saying nothing. + """ + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + dialect = self._dialect() + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.format_data_type(build(dialect, True)) + suggestion = excinfo.value.suggestion or "" + assert "CONSTRAINT" in suggestion, suggestion + assert "469" in suggestion, suggestion + assert "syntax error" in suggestion, suggestion + + @pytest.mark.parametrize("label,klass,build,signed_sql,word", + _FLOATS_AND_DECIMALS, ids=_IDS) + def test_the_signed_form_is_byte_identical_to_what_it_was( + self, label, klass, build, signed_sql, word): + dialect = self._dialect() + assert dialect.format_data_type(build(dialect, False)) == (signed_sql, ()) + # The bare declaration is a different column for ``decimal`` — this dialect + # fills its own ``Decimal(10, 0)`` default in — and the same one for the + # three floats, so it is compared against its own rendering rather than the + # builder's. + bare = dialect.format_data_type(klass(dialect))[0] + assert bare == ("Decimal(10, 0)" if label == "decimal" else signed_sql) + + @pytest.mark.parametrize("label,klass,build,signed_sql,word", + _FLOATS_AND_DECIMALS, ids=_IDS) + def test_signedness_still_participates_in_equality(self, label, klass, build, + signed_sql, word): + """Why refusing is the right answer: the two declarations are genuinely + different types, so the differ can see a caller changed their mind. Refusing + at render time stops the DDL lying about that; it must not flatten the + model instead.""" + dialect = self._dialect() + signed, unsigned = build(dialect, False), build(dialect, True) + assert signed != unsigned + assert hash(signed) != hash(unsigned) + + def test_unsigned_is_appended_so_hashes_are_stable(self): + """Appended, never inserted: the order of ``PARAMETERS`` is what + ``identity()`` reads, so it is what ``__eq__`` and ``__hash__`` read.""" + assert DecimalType.PARAMETERS == ("precision", "scale", "unsigned") + assert FloatType.PARAMETERS == ("precision", "unsigned") + assert DoubleType.PARAMETERS == ("unsigned",) + assert RealType.PARAMETERS == ("unsigned",) + + def test_the_native_float_and_decimal_classes_cannot_be_given_the_field(self): + """Why the refusal lives on the four core concepts and not on + ``clickhouse_float32`` / ``clickhouse_float64`` / ``clickhouse_decimal*``. + + Those classes narrow ``PARAMETERS`` to ``()`` and their ``__init__`` takes + no ``unsigned``, so the flag is *pinned* — there is no reachable + declaration to drop. The core concepts are the four a caller reaches by the + plain SQL words ``DECIMAL`` / ``FLOAT`` / ``DOUBLE`` / ``REAL``, and those + are the four that must answer. + """ + dialect = self._dialect() + for name in ("ClickHouseFloat32Type", "ClickHouseFloat64Type", + "ClickHouseDecimal32Type", "ClickHouseDecimal64Type", + "ClickHouseDecimal128Type"): + klass = getattr(clickhouse_types, name) + assert "unsigned" not in klass.PARAMETERS, name + assert klass(dialect).unsigned is False, name + with pytest.raises(TypeError): + clickhouse_types.ClickHouseFloat32Type(dialect, unsigned=True) + + def test_a_type_string_carrying_unsigned_is_refused_rather_than_read(self): + """The same rule in the *reading* direction. + + Every numeric branch of ``parse_type`` matches on the word in front of the + attribute, so without a check ``parse_type("Float32 UNSIGNED")`` was a signed + ``ClickHouseFloat32Type`` and ``parse_type("Decimal(10,2) UNSIGNED")`` was + ``ClickHouseDecimalType(10, 2)`` — value objects ``==`` calls equal to the + signed column, so the differ reported no change for the one change the + string describes. + + The MySQL-style integer aliases ClickHouse **does** accept (``TINYINT + UNSIGNED`` and friends, which the server resolves to ``UInt8``…``UInt64``) + are spellings of a column rather than an attribute tacked onto one, so they + are read as the ``UInt`` family they name — see + :meth:`test_the_mysql_unsigned_aliases_are_read_as_the_uint_family` — and + this refusal must not swallow them. + """ + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + dialect = self._dialect() + for raw in ("Float32 UNSIGNED", "Float64 UNSIGNED", + "Decimal(10,2) UNSIGNED", "Decimal(10, 2) UNSIGNED", + "Int8 UNSIGNED", "Int32 UNSIGNED"): + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.parse_type(raw) + assert "UNSIGNED" in str(excinfo.value), raw + # ... and the unsigned-free strings of the same shapes are untouched. + assert dialect.parse_type("Float32") == clickhouse_types.ClickHouseFloat32Type( + dialect) + assert dialect.parse_type("Decimal(10, 2)") == ( + clickhouse_types.ClickHouseDecimalType(dialect, precision=10, scale=2)) + # ... and a real UInt column still reads back as unsigned. + assert dialect.parse_type("UInt8").unsigned is True + + #: ``(alias, class, the native word it renders)``. Every alias is + #: ``case_insensitive = 1`` in ``system.data_type_families`` with the matching + #: ``alias_to``, measured on 26.7.3.19; the live half of the check is + #: ``types/test_native_types.py::TestSignednessIsInTheTypeName``. + _UNSIGNED_ALIASES = [ + ("TINYINT UNSIGNED", ClickHouseUInt8Type, "UInt8"), + ("INT1 UNSIGNED", ClickHouseUInt8Type, "UInt8"), + ("SMALLINT UNSIGNED", ClickHouseUInt16Type, "UInt16"), + ("YEAR", ClickHouseUInt16Type, "UInt16"), + ("MEDIUMINT UNSIGNED", ClickHouseUInt32Type, "UInt32"), + ("INT UNSIGNED", ClickHouseUInt32Type, "UInt32"), + ("INTEGER UNSIGNED", ClickHouseUInt32Type, "UInt32"), + ("BIGINT UNSIGNED", ClickHouseUInt64Type, "UInt64"), + ("UNSIGNED", ClickHouseUInt64Type, "UInt64"), + ] + _UNSIGNED_ALIAS_IDS = [row[0] for row in _UNSIGNED_ALIASES] + + @pytest.mark.parametrize("raw, klass, rendered", _UNSIGNED_ALIASES, + ids=_UNSIGNED_ALIAS_IDS) + def test_the_mysql_unsigned_aliases_are_read_as_the_uint_family( + self, raw, klass, rendered): + """One concept, one class: every alias yields the class that owns the column. + + ``parse_type("TINYINT UNSIGNED")`` used to raise through the ``CustomType`` + validation, because a two-word name is not in the core grammar's multi-word + list. The server accepts it and resolves it to ``UInt8``, so the parser + hands back ``ClickHouseUInt8Type`` — not a synonym, not the signed class — + and ``format_data_type`` writes the native spelling. That is what makes an + aliased DDL string equal to the column it creates. + """ + dialect = self._dialect() + parsed = dialect.parse_type(raw) + assert parsed == klass(dialect), raw + assert parsed.unsigned is True, raw + assert dialect.format_data_type(parsed)[0] == rendered, raw + # The server records these as case-insensitive; the parser upper-cases, + # so the lower-case spelling must reach the same class. + assert dialect.parse_type(raw.lower()) == klass(dialect), raw + + @pytest.mark.parametrize("case,expected", [ + # the pre-existing ValueErrors, still firing for a *signed* declaration + ("decimal_precision", ValueError), + ("decimal_scale", ValueError), + ]) + def test_every_pre_existing_check_still_fires_for_a_signed_declaration( + self, case, expected): + """Nothing moved except the signedness refusal. + + The two bounds below live in ``ClickHouseDecimalType.__init__`` rather than + in a formatter — this backend fixes the precision at the width and validates + it where the value enters — so the ``raises`` block wraps the construction, + which is where the ``ValueError`` has always come from. + """ + dialect = self._dialect() + with pytest.raises(expected): + { + "decimal_precision": clickhouse_types.ClickHouseDecimalType( + dialect, precision=99), + "decimal_scale": clickhouse_types.ClickHouseDecimalType( + dialect, precision=10, scale=99), + }[case] + + def test_the_signedness_gate_runs_before_the_spelling_gate(self): + """A request wrong in two ways is told about the right one first. + + Signedness is a declaration this grammar cannot express **at all**, which is + a stronger and less recoverable statement than a misspelled word, so the + refusal comes first. + """ + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + dialect = self._dialect() + for built in (DecimalType(dialect, 10, 2, spelling="fixed", unsigned=True), + DoubleType(dialect, spelling="float", unsigned=True)): + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.format_data_type(built) + assert "unsigned" in str(excinfo.value).lower() + + def test_the_refusal_is_not_a_value_error(self): + """The cross-backend exception convention, stated as a test: a wrong value + is ``ValueError``; a declaration this grammar cannot express at all is + ``UnsupportedFeatureError``; the two do not share a base class.""" + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + assert not issubclass(UnsupportedFeatureError, ValueError) + with pytest.raises(UnsupportedFeatureError): + self._dialect().format_data_type( + DecimalType(self._dialect(), unsigned=True)) + + +# ── The vector type renders ClickHouse's word, not MySQL's ───────────── + +class TestVectorRendersQBit: + """``ClickHouseVectorType`` renders ``QBit(Float32, n)``, not ``VECTOR(n)``. + + ``VECTOR(n)`` is **MySQL 9.0's** spelling and was carried into this backend + together with MySQL's ``STRING_TO_VECTOR`` / ``VECTOR_DIM`` / ``DISTANCE_*`` + family. ClickHouse has no such type: on the 26.7 scenario server + ``CREATE TABLE t (a VECTOR(4))`` returns + ``Unknown data type family: VECTOR``, and ``system.data_type_families`` + lists ``QBit`` and no ``VECTOR`` entry. ClickHouse's own vector type is + ``QBit(element_type, dimension[, stride])`` — + https://clickhouse.com/docs/reference/data-types/qbit + """ + + def test_rendered_sql_is_the_documented_qbit_spelling(self): + dialect = ClickHouseDialect(version=(26, 7, 0)) + sql, params = ClickHouseVectorType(dialect, dim=4).to_sql() + assert sql == "QBit(Float32, 4)" + assert params == () + + @pytest.mark.parametrize("dim", [3, 4, 128, 1536]) + def test_dimension_is_carried_through(self, dim): + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert dialect.format_data_type( + ClickHouseVectorType(dialect, dim=dim))[0] == f"QBit(Float32, {dim})" + + def test_it_does_not_render_the_mysql_spelling(self): + """The regression this whole class exists for. + + Rendering ``VECTOR(n)`` produced DDL the server refuses, which is worse + than not supporting the type at all: it fails at ``CREATE TABLE`` time, on + a statement the framework itself built. + """ + dialect = ClickHouseDialect(version=(26, 7, 0)) + sql, _ = ClickHouseVectorType(dialect, dim=4).to_sql() + assert "VECTOR" not in sql.upper() + + # ── version gate ──────────────────────────────────────────────────── + # + # QBit appeared in 25.10 behind ``allow_experimental_qbit_type``, was promoted + # to Beta in 26.1 and to General Availability in 26.2. The gate is GA: before + # 26.2 a column declaration still needs an experimental opt-in that DDL does + # not carry. + + @pytest.mark.parametrize("version", [(26, 2, 0), (26, 3, 0), (26, 7, 3)]) + def test_supported_from_ga_onwards(self, version): + dialect = ClickHouseDialect(version=version) + assert dialect.format_data_type( + ClickHouseVectorType(dialect, dim=8))[0] == "QBit(Float32, 8)" + + @pytest.mark.parametrize("version", [(25, 8, 0), (25, 10, 0), (26, 1, 0)]) + def test_refused_before_ga(self, version): + from rhosocial.activerecord.backend.dialect.exceptions import ( + UnsupportedFeatureError, + ) + dialect = ClickHouseDialect(version=version) + with pytest.raises(UnsupportedFeatureError) as excinfo: + dialect.format_data_type(ClickHouseVectorType(dialect, dim=8)) + message = str(excinfo.value) + assert "QBit" in message + assert "Array(Float32)" in message, "the error must say what to use instead" + + def test_unadapted_dialect_refuses_rather_than_guessing(self): + """No version means no guarantee, and an unguessed guess is the defect. + + The dialect has not talked to a server, so it cannot know whether ``QBit`` + is available — emitting it blind would reintroduce exactly the failure + this fix removes. + """ + from rhosocial.activerecord.backend.dialect.exceptions import ( + DialectNotAdaptedException, + ) + dialect = ClickHouseDialect() + with pytest.raises(DialectNotAdaptedException): + dialect.format_data_type(ClickHouseVectorType(dialect, dim=8)) + + # ── parse_type, which is what introspection actually calls ────────── + + @pytest.mark.parametrize("raw", [ + "QBit(Float32, 8)", + "QBit(Float64, 1536)", + "QBit(BFloat16, 5)", + "QBit(Int8, 2048)", + ]) + def test_parse_reads_the_dimension_not_the_element_type(self, raw): + """The server reports ``QBit(Float32, 8)`` — checked with DESCRIBE on 26.7. + + The obvious implementation, scanning for the first digit run, reads the + ``32`` out of ``Float32`` and reports an 8-dimensional column as + 32-dimensional. The dimension is the first argument that is a *bare* + integer, which is exactly why the element type must be skipped rather + than parsed. + """ + dialect = ClickHouseDialect(version=(26, 7, 0)) + expected = int(raw[raw.rindex(",") + 1: raw.rindex(")")]) + assert dialect.parse_type(raw) == ClickHouseVectorType(dialect, dim=expected) + + def test_parse_keeps_the_dimension_of_a_strided_qbit(self): + """``stride`` was added in 26.7; it is a third argument, not the dimension.""" + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert dialect.parse_type("QBit(Float32, 16, 8)") == \ + ClickHouseVectorType(dialect, dim=16) + + @pytest.mark.parametrize("dim", [8, 1536]) + def test_parse_then_render_is_a_fixed_point(self, dim): + dialect = ClickHouseDialect(version=(26, 7, 0)) + raw = f"QBit(Float32, {dim})" + assert dialect.format_data_type(dialect.parse_type(raw))[0] == raw + + @pytest.mark.parametrize("element_type", ["BFloat16", "Float64", "Int8"]) + def test_parse_loses_the_element_type(self, element_type): + """Known lossiness, asserted so it stays a decision. + + ``QBit``'s element type is part of its SQL form but not part of + :class:`ClickHouseVectorType`'s identity, so a column introspected as + ``QBit(BFloat16, 1536)`` comes back as ``QBit(Float32, 1536)``. The + *dimension* — the part this class does model, and the part that cannot + change without rewriting the column — round-trips exactly. Carrying the + element type would mean making it part of the class's identity, which is + a wider change than fixing a wrong spelling; reported rather than fixed. + """ + dialect = ClickHouseDialect(version=(26, 7, 0)) + parsed = dialect.parse_type(f"QBit({element_type}, 1536)") + assert parsed == ClickHouseVectorType(dialect, dim=1536) + assert dialect.format_data_type(parsed)[0] == "QBit(Float32, 1536)" + + def test_mysql_vector_spelling_is_not_a_clickhouse_type(self): + """``VECTOR(4)`` is not ClickHouse's, so ``parse_type`` must not claim it. + + It falls through to ``CustomType``, which cannot be rendered — the same + treatment an unclaimed word gets. ``VARCHAR(10)`` is the sharper + example: the server *accepts* it (it resolves to ``String``), but this + parser does not claim it, so it too falls to ``CustomType``. + """ + from rhosocial.activerecord.backend.expression.types import CustomType + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert isinstance(dialect.parse_type("VECTOR(4)"), CustomType) + + +# ── the rest of MySQL's vector API is not here at all ───────────────── +# +# ``VECTOR(n)`` is only the visible end of the imported surface. The same commit +# brought ``STRING_TO_VECTOR`` / ``VECTOR_TO_STRING`` / ``VECTOR_DIM`` / +# ``DISTANCE_*`` and a ``MAX_VECTOR_DIMENSION = 16384`` ceiling, all MySQL's. On +# the 26.7 scenario server none of the functions exists — each answers +# ``Function with name `...` does not exist`` — and 16384 is MySQL's limit for a +# type ClickHouse does not have. The surface is therefore deleted rather than +# stubbed; the live half of that evidence is in ``types/test_native_types.py``. + +MYSQL_VECTOR_FUNCTIONS = ( + "STRING_TO_VECTOR", + "VECTOR_TO_STRING", + "VECTOR_DIM", + "DISTANCE_EUCLIDEAN", + "DISTANCE_COSINE", + "DISTANCE_DOT", +) + +#: The dialect methods the deleted ``ClickHouseVectorSupport`` / mixin declared. +MYSQL_VECTOR_DIALECT_METHODS = ( + "supports_vector_type", + "supports_vector_index", + "get_max_vector_dimension", + "format_vector_literal", + "format_string_to_vector", + "format_vector_to_string", + "format_vector_dim", + "format_distance_euclidean", + "format_distance_cosine", + "format_distance_dot", + "format_create_vector_index", +) + + +class TestMySQLVectorSurfaceIsGone: + """The MySQL surface is **absent**, not stubbed. + + Keeping it as a fail-fast stub was the state this class replaces, and it had a + defect beyond being dead weight: ``supports_vector_type()`` answered ``False`` + on the same dialect where ``supports_data_type_clickhouse_vector()`` answers + ``True`` and renders ``QBit(Float32, n)``. Two capability flags on one dialect + disagreeing about whether ClickHouse has a vector type is worse than silence — + ``get_max_vector_dimension()`` then returned MySQL's 16384 for a type this + backend does not have. + + ClickHouse's real vector functions are ``L2Distance``, ``cosineDistance``, + ``dotProduct`` (and the ``*Transposed`` trio for ``QBit`` columns): + https://clickhouse.com/docs/reference/functions/regular-functions/distance-functions + """ + + @pytest.mark.parametrize("method", MYSQL_VECTOR_DIALECT_METHODS) + def test_the_dialect_has_no_such_method(self, method): + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert not hasattr(dialect, method), ( + f"{method} is MySQL's; ClickHouse's vector type is QBit" + ) + + def test_no_protocol_declares_it(self): + from rhosocial.activerecord.backend.impl.clickhouse import protocols + assert not [name for name in vars(protocols) if "Vector" in name] + assert not hasattr(protocols, "ClickHouseVectorSupport") + + def test_no_mixin_implements_it(self): + from rhosocial.activerecord.backend.impl.clickhouse import mixins + assert not hasattr(mixins, "ClickHouseVectorMixin") + assert not [name for name in vars(mixins) if "VectorMixin" in name] + + @pytest.mark.parametrize("function_name", MYSQL_VECTOR_FUNCTIONS) + def test_no_expression_node_claims_to_render_it(self, function_name): + """No expression class advertises a ``format_method`` the dialect cannot have. + + The deleted nodes each returned e.g. ``"format_string_to_vector"`` from + ``format_method``; a node whose dispatcher name the dialect does not define + is a node that fails with ``AttributeError`` rather than with an + ``UnsupportedFeatureError`` that says what to use. + """ + from rhosocial.activerecord.backend.impl.clickhouse import expression as expr + + wanted = "format_" + function_name.lower() + offending = [] + for name in expr.__all__: + klass = getattr(expr, name, None) + if not (isinstance(klass, type) and hasattr(klass, "format_method")): + continue + # every concrete getter returns a literal, so the class is a + # sufficient stand-in for an instance; an abstract one raises + try: + dispatcher = klass.format_method.fget(klass) + except NotImplementedError: + continue + if dispatcher == wanted: + offending.append(name) + assert not offending, offending + + def test_the_capability_flag_no_longer_contradicts_itself(self): + """The concept this backend *does* support keeps answering for itself.""" + dialect = ClickHouseDialect(version=(26, 7, 0)) + assert dialect.supports_data_type_clickhouse_vector() is True + assert dialect.format_data_type( + ClickHouseVectorType(dialect, dim=4))[0] == "QBit(Float32, 4)" + + +# --------------------------------------------------------------------------- +# parse_type: the sweep, and the two words it found unreadable +# --------------------------------------------------------------------------- + + +class TestGeometryCollectionIsNotTheGenericGeometry: + """``GEOMETRYCOLLECTION`` begins with ``GEOMETRY``, and the map put + + ``GEOMETRY`` first, so the ``startswith`` loop read a collection column + as the generic geometry type -- the same defect MariaDB's and MySQL's + branches of this family each found and fixed, found here by the + render-parse-re-render sweep. Dispatch is on the leading word now, + which is the whole name for every one of the eight words. + """ + + def test_a_collection_is_not_the_generic_geometry_type(self): + from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( + ClickHouseGeometryCollectionType, + ClickHouseGeometryType, + ) + + dialect = ClickHouseDialect() + collection = dialect.parse_type("GEOMETRYCOLLECTION") + generic = dialect.parse_type("GEOMETRY") + assert isinstance(collection, ClickHouseGeometryCollectionType) + assert collection != generic + assert type(collection) is not type(generic) + + @pytest.mark.parametrize("word,expected", [ + ("POINT", "ClickHousePointType"), + ("LINESTRING", "ClickHouseLineStringType"), + ("POLYGON", "ClickHousePolygonType"), + ("MULTIPOINT", "ClickHouseMultiPointType"), + ("MULTILINESTRING", "ClickHouseMultiLineStringType"), + ("MULTIPOLYGON", "ClickHouseMultiPolygonType"), + ("GEOMETRYCOLLECTION", "ClickHouseGeometryCollectionType"), + ]) + def test_each_word_reads_back_as_its_own_shape(self, word, expected): + dialect = ClickHouseDialect() + assert type(dialect.parse_type(word)).__name__ == expected, word + + def test_srid_survives_the_dispatch(self): + from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( + ClickHouseGeometryCollectionType, + ) + + dialect = ClickHouseDialect() + parsed = dialect.parse_type("GEOMETRYCOLLECTION SRID 4326") + assert isinstance(parsed, ClickHouseGeometryCollectionType) + assert parsed.srid == 4326 + + +class TestTupleElementsParseBack: + """``Tuple(name Type, ...)`` -- the field names are part of the type. + + Live evidence on ClickHouse 26.7.3.19: the names are stored and + reported (``system.columns.type`` / ``SHOW CREATE`` / ``DESCRIBE`` / + ``toTypeName``), compared and hashed, used for ``tuple.name`` access, + and matched during INSERT conversion -- see the class docstring in + ``expression/types.py`` and the report's section A. The parse has to + read them: handing ``"name Type"`` to ``parse_type`` whole fell through + to ``CustomType``, whose type-name validation refused the space, so the + dialect raised on its own rendering. The leading identifier is split + off when a second word follows, and carried into ``element_names`` + because the class records them in PARAMETERS -- the assertions below + pin that two tuples differing only in their names are different types. + """ + + def test_a_named_tuple_parses_back(self): + from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( + ClickHouseTupleType, + ClickHouseInt32Type, + ) + + dialect = ClickHouseDialect() + parsed = dialect.parse_type("Tuple(a Int32)") + assert isinstance(parsed, ClickHouseTupleType) + assert parsed.element_types == (ClickHouseInt32Type(dialect),) + assert parsed.element_names == ("a",) + + def test_a_bare_positional_tuple_parses_back(self): + from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( + ClickHouseTupleType, + ) + dialect = ClickHouseDialect() - from rhosocial.activerecord.backend.expression.types import UUIDType - with pytest.raises(TypeError, match="does not support"): - UUIDType(dialect).to_sql() + parsed = dialect.parse_type("Tuple(Int32, String)") + assert isinstance(parsed, ClickHouseTupleType) + assert len(parsed.element_types) == 2 + assert parsed.element_names is None + + def test_the_named_render_round_trips(self): + from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( + ClickHouseTupleType, + ClickHouseInt32Type, + ) + + dialect = ClickHouseDialect() + declared = ClickHouseTupleType( + dialect, + element_types=[ClickHouseInt32Type(dialect)], + element_names=["a"], + ) + sql, _ = declared.to_sql() + assert sql == "Tuple(a Int32)" + assert dialect.parse_type(sql) == declared + + def test_tuples_differing_only_in_element_names_are_different(self): + """The names are identity, not documentation. + + ``Tuple(a Int32)`` and ``Tuple(b Int32)`` are different columns to + the server, and the differ relies on this inequality to emit the + renaming ``ALTER ... MODIFY COLUMN``; a tuple with no names is + different from either named one. + """ + from rhosocial.activerecord.backend.impl.clickhouse.expression.types import ( + ClickHouseTupleType, + ClickHouseInt32Type, + ) + + dialect = ClickHouseDialect() + element = ClickHouseInt32Type(dialect) + named_a = ClickHouseTupleType( + dialect, element_types=[element], element_names=["a"], + ) + named_b = ClickHouseTupleType( + dialect, element_types=[element], element_names=["b"], + ) + unnamed = ClickHouseTupleType(dialect, element_types=[element]) + assert named_a != named_b + assert named_a != unnamed + assert hash(named_a) != hash(named_b) + assert hash(named_a) != hash(unnamed) + + +class TestParseRoundTripSweep: + """The shared sweep across this backend's whole declared surface. + + Asserts the two invariants at once: **string stability** -- parsing a + rendering and re-rendering the answer produces the identical string -- + and **class honesty** -- the answer is the declared instance, or the + documented widening answer recorded below. Every answer here is + string-stable; the widening table maps each core concept onto the + ClickHouse class of the storage it renders (Int8..Int64, Float32/64, + String for the text and byte families, DateTime for the temporal ones), + and the backend's own classes parse back to themselves. + """ + + #: Core concept -> the backend class of the storage it renders here. + WIDENING_ANSWERS = { + "ArrayType": "ClickHouseArrayType", + "BlobType": "ClickHouseStringType", + "BooleanType": "ClickHouseBoolType", + "DateTimeType": "ClickHouseDateTimeType", + "DateType": "ClickHouseDateType", + "TimeType": "ClickHouseDateTimeType", + "TimestampType": "ClickHouseDateTimeType", + "BigIntType": "ClickHouseInt64Type", + "IntegerType": "ClickHouseInt32Type", + "SmallIntType": "ClickHouseInt16Type", + "TinyIntType": "ClickHouseInt8Type", + "JsonType": "ClickHouseStringType", + "DecimalType": "ClickHouseDecimalType", + "DoubleType": "ClickHouseFloat64Type", + "FloatType": "ClickHouseFloat32Type", + "RealType": "ClickHouseFloat32Type", + "CharType": "ClickHouseStringType", + "TextType": "ClickHouseStringType", + "VarCharType": "ClickHouseStringType", + "UUIDType": "ClickHouseUUIDType", + # The backend's own classes: same class back, the parameter + # normalisation the parse applies (tuples for lists) is the only + # difference, and the re-render is byte-identical. + "ClickHouseAggregateFunctionType": "ClickHouseAggregateFunctionType", + "ClickHouseArrayType": "ClickHouseArrayType", + "ClickHouseLowCardinalityType": "ClickHouseLowCardinalityType", + "ClickHouseMapType": "ClickHouseMapType", + "ClickHouseNullableType": "ClickHouseNullableType", + "ClickHouseSimpleAggregateFunctionType": "ClickHouseSimpleAggregateFunctionType", + "ClickHouseTupleType": "ClickHouseTupleType", + } + + @pytest.fixture(scope="class") + def registry(self): + """The round-trip module's registry, loaded from the parent + + directory. Executing it also registers its special constructors, + which the sweep's ``make_instance`` consults -- the same + registrations the full suite performs at collection time. + """ + import importlib.util + from pathlib import Path + + path = Path(__file__).resolve().parent.parent / ( + "test_expression_roundtrip_all.py" + ) + spec = importlib.util.spec_from_file_location( + "clickhouse_rt_registry", + path, + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + registry = getattr(module, "ALL_CLASSES", None) or module.REGISTERED + assert registry, "the round-trip module exposes no registry" + return registry + + def test_every_rendered_type_parses_back_coherently(self, registry): + from rhosocial.activerecord.testsuite.utils.parse_contract import ( + parse_roundtrip_failures, + ) + + dialect = ClickHouseDialect() + failures = parse_roundtrip_failures( + dialect, + registry, + widening=self.WIDENING_ANSWERS, + ) + assert failures == [], "\n".join(failures) + diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_native_types.py b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_native_types.py index 7f20848..0a81c9c 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_native_types.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/backend/types/test_native_types.py @@ -160,3 +160,232 @@ def test_clickhouse_dialect_supports_native_types(self, clickhouse_backend): assert d.supports_map_type() is True if hasattr(d, "supports_map_type") else True assert d.supports_json_type() is True assert d.supports_microsecond_timestamp() is True + + +# ── the server is the authority on which integers and vectors exist ──── +# +# The unit half of this pair lives in ``test_clickhouse_type_protocol.py``. These +# tests ask the scenario server directly, because both decisions are claims about +# ClickHouse's *own* type inventory rather than about this dialect's code: +# +# * ``unsigned`` is refused on the signed integer classes because ClickHouse +# spells signedness in the type name — so ``Int8 UNSIGNED`` must be a parse +# error, and the unsigned form of an integer must be spelled ``UInt8``. +# * MySQL's ``VECTOR`` type and its function family are gone from this backend +# because ClickHouse has neither — so both claims are checked against the server. + +QBIT_GA = (26, 2, 0) + +#: MySQL-style unsigned integer aliases ClickHouse accepts, and the ``UInt`` +#: family the server resolves each to. Measured on the 26.7.3.19 scenario +#: server: every one is ``case_insensitive = 1`` in +#: ``system.data_type_families`` with the matching ``alias_to``, and +#: ``SELECT toTypeName(CAST(1 AS ))`` returns the family. ``YEAR`` is +#: ClickHouse's own alias for ``UInt16`` — the one-byte MySQL ``YEAR`` concept +#: lives in the MySQL backend, not here. +UNSIGNED_INTEGER_ALIASES = ( + ("TINYINT UNSIGNED", "UInt8"), + ("INT1 UNSIGNED", "UInt8"), + ("SMALLINT UNSIGNED", "UInt16"), + ("YEAR", "UInt16"), + ("MEDIUMINT UNSIGNED", "UInt32"), + ("INT UNSIGNED", "UInt32"), + ("INTEGER UNSIGNED", "UInt32"), + ("BIGINT UNSIGNED", "UInt64"), + ("UNSIGNED", "UInt64"), +) + + +class TestSignednessIsInTheTypeName: + """ClickHouse's integer grammar, verified. + + https://clickhouse.com/docs/sql-reference/data-types/int-uint + """ + + @pytest.mark.parametrize("alias, expected", UNSIGNED_INTEGER_ALIASES) + def test_the_unsigned_form_is_a_different_type_name(self, clickhouse_backend, alias, expected): + """The aliases exist, and they resolve to ``UInt*``. + + This is why honouring ``unsigned`` on ``ClickHouseInt8Type`` was refused + rather than accepted: the rendering that would have honoured it is a column + byte-for-byte identical to the one ``ClickHouseUInt8Type`` already produces. + """ + row = clickhouse_backend.fetch_one(f"SELECT toTypeName(CAST(1 AS {alias})) AS t") + assert row["t"] == expected + + @pytest.mark.parametrize("alias, expected", UNSIGNED_INTEGER_ALIASES) + def test_parse_type_reads_each_alias_as_the_class_the_server_names( + self, clickhouse_backend, alias, expected): + """The parser and the server have to agree on the mapping. + + ``parse_type`` is the reading direction of the type system, so a live + server agreeing with the formatter is only half the loop. The parsed + value must render as the family ``toTypeName`` returns and must be the + ``ClickHouseUInt*Type`` that owns the column — the alias is + canonicalised to the native spelling, never to a synonym or to the + signed class. + """ + backend = clickhouse_backend + row = backend.fetch_one(f"SELECT toTypeName(CAST(1 AS {alias})) AS t") + parsed = backend.dialect.parse_type(alias) + assert type(parsed).__name__ == f"ClickHouse{expected}Type", alias + assert backend.dialect.format_data_type(parsed)[0] == row["t"] == expected + assert parsed.unsigned is True + + def test_unsigned_is_not_a_modifier_on_the_native_name(self, clickhouse_backend): + """``Int8 UNSIGNED`` is not a spelling, so there is nothing to write. + + The backend reports ``Unknown data type family: INT8 UNSIGNED``, which is + the reason ``format_data_type_clickhouse_int8`` raises rather than + appending anything. + """ + from rhosocial.activerecord.backend.errors import DatabaseError + + with pytest.raises(DatabaseError) as excinfo: + clickhouse_backend.execute( + "CREATE TABLE test_ch_unsigned (a Int8 UNSIGNED) ENGINE = Memory" + ) + assert "UNSIGNED" in str(excinfo.value).upper() + + def test_the_rendered_words_are_the_two_distinct_types(self, clickhouse_backend): + """``Int8`` and ``UInt8`` are two types, not one type and a modifier. + + Asserted through ``system.columns`` so the formatter's output and the + server's own view of the column are tied together: the words + ``format_data_type_clickhouse_int8`` and + ``format_data_type_clickhouse_uint8`` emit are two different rows here. + + (ClickHouse wraps an out-of-range value on ``INSERT`` rather than refusing + it — ``-1`` into a ``UInt8`` lands as ``255`` — so the ranges are asserted + as documented type boundaries, not as insert-time rejection.) + """ + backend = clickhouse_backend + backend.execute("DROP TABLE IF EXISTS test_ch_signedness") + backend.execute(""" + CREATE TABLE test_ch_signedness ( + s Int8, + u UInt8 + ) ENGINE = Memory + """) + try: + rows = backend.fetch_all( + "SELECT name, type FROM system.columns " + "WHERE table = 'test_ch_signedness' ORDER BY name" + ) + declared = {row["name"]: row["type"] for row in rows} + assert declared == {"s": "Int8", "u": "UInt8"} + assert backend.fetch_one("SELECT toInt8(-128) AS v")["v"] == -128 + assert backend.fetch_one("SELECT toUInt8(255) AS v")["v"] == 255 + finally: + backend.execute("DROP TABLE IF EXISTS test_ch_signedness") + + +class TestMysqlVectorSurfaceDoesNotExistOnTheServer: + """The evidence behind deleting that surface, checked against the server. + + Every statement here is expected to fail. If one of them starts succeeding, + ClickHouse has grown the concept and the deletion should be revisited. + """ + + @pytest.mark.parametrize("ddl", [ + "CREATE TABLE test_ch_vec (a VECTOR(4)) ENGINE = Memory", + ]) + def test_there_is_no_vector_column_type(self, clickhouse_backend, ddl): + from rhosocial.activerecord.backend.errors import DatabaseError + + with pytest.raises(DatabaseError) as excinfo: + clickhouse_backend.execute(ddl) + assert "VECTOR" in str(excinfo.value).upper() + clickhouse_backend.execute("DROP TABLE IF EXISTS test_ch_vec") + + @pytest.mark.parametrize("call", [ + "STRING_TO_VECTOR('[1,2,3]')", + "VECTOR_TO_STRING([1.0,2.0,3.0])", + "VECTOR_DIM([1.0,2.0,3.0])", + "DISTANCE_EUCLIDEAN([1.0,2.0],[1.0,3.0])", + "DISTANCE_COSINE([1.0,2.0],[1.0,3.0])", + "DISTANCE_DOT([1.0,2.0],[1.0,3.0])", + ]) + def test_there_are_no_mysql_vector_functions(self, clickhouse_backend, call): + from rhosocial.activerecord.backend.errors import DatabaseError + + with pytest.raises(DatabaseError) as excinfo: + clickhouse_backend.execute(f"SELECT {call}") + message = str(excinfo.value) + assert "does not exist" in message, message + + def test_the_replacement_functions_do_exist(self, clickhouse_backend): + """What the guidance names must actually be callable. + + ``L2Distance`` / ``cosineDistance`` / ``dotProduct`` are ClickHouse's own + distance functions: + https://clickhouse.com/docs/reference/functions/regular-functions/distance-functions + """ + for call, expected in ( + ("L2Distance([1.0,2.0],[1.0,3.0])", 1.0), + ("dotProduct([1.0,2.0],[1.0,3.0])", 7.0), + ): + row = clickhouse_backend.fetch_one(f"SELECT {call} AS d") + assert abs(row["d"] - expected) < 1e-9, call + row = clickhouse_backend.fetch_one("SELECT cosineDistance([1.0,2.0],[1.0,3.0]) AS d") + assert 0.0 <= row["d"] <= 1.0 + + def test_the_max_vector_dimension_constant_is_not_a_clickhouse_number(self, clickhouse_backend): + """``16384`` was MySQL's ceiling; ClickHouse states no such limit. + + ``QBit``'s ``dimension`` argument is bounded by what the server can address, + not by a vendor maximum, which is why the deleted + ``get_max_vector_dimension()`` had nothing honest to return. + """ + row = clickhouse_backend.fetch_one( + "SELECT count() AS c FROM system.data_type_families WHERE name ILIKE '%ECTOR%'" + ) + assert row["c"] == 0 + + +class TestQBitIsClickhousesOwnVectorType: + """The rendering this backend does ship, exercised against the server. + + https://clickhouse.com/docs/sql-reference/data-types/qbit + """ + + def test_qbit_column_roundtrips(self, clickhouse_backend): + backend = clickhouse_backend + if backend.dialect.version < QBIT_GA: + pytest.skip(f"QBit needs ClickHouse {QBIT_GA}; this is {backend.dialect.version}") + backend.execute("DROP TABLE IF EXISTS test_ch_qbit") + backend.execute(""" + CREATE TABLE test_ch_qbit ( + id UInt32, + vec QBit(Float32, 4) + ) ENGINE = MergeTree() + ORDER BY id + """) + try: + backend.execute( + "INSERT INTO test_ch_qbit VALUES (%s, %s)", + (1, [1.0, 2.0, 3.0, 4.0]), + ) + row = backend.fetch_one("SELECT vec FROM test_ch_qbit") + assert [float(v) for v in row["vec"]] == [1.0, 2.0, 3.0, 4.0] + reported = backend.fetch_one( + "SELECT type FROM system.columns " + "WHERE table = 'test_ch_qbit' AND name = 'vec'" + ) + assert reported["type"] == "QBit(Float32, 4)" + finally: + backend.execute("DROP TABLE IF EXISTS test_ch_qbit") + + def test_the_dimension_is_part_of_the_column(self, clickhouse_backend): + """Two dimensions are two columns, which is why ``dim`` is in ``PARAMETERS``. + + A column cannot change its dimension in place: ``QBit``'s dimension is part + of its type, and a ``SELECT`` over a mismatched width is a type error. + """ + backend = clickhouse_backend + if backend.dialect.version < QBIT_GA: + pytest.skip(f"QBit needs ClickHouse {QBIT_GA}; this is {backend.dialect.version}") + with pytest.raises(Exception): + backend.execute( + "SELECT [1.0,2.0,3.0,4.0]::Array(Float32)::QBit(Float32, 3)" + ) diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field.py b/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field.py index 540631b..40ecb63 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field.py @@ -6,6 +6,7 @@ from rhosocial.activerecord.base import DerivedField, UseColumn from rhosocial.activerecord.model import ActiveRecord from rhosocial.activerecord.backend.expression import Column, Literal +from rhosocial.activerecord.backend.expression.operators import BinaryArithmeticExpression class TestDerivedFieldDeclaration: @@ -20,7 +21,9 @@ def test_descriptor_class_access(self, product_class): def test_derived_field_with_static_expression(self, product_class): """Test DerivedField constructed with a pre-built expression object (non-callable path).""" dialect = product_class.backend().dialect - static_expr = Column(dialect, "price") * Literal(dialect, 2) + static_expr = BinaryArithmeticExpression( + dialect, "*", Column(dialect, "price"), Literal(dialect, 2) + ) df = DerivedField(static_expr) resolved = df.resolve(dialect) assert resolved is static_expr @@ -137,7 +140,11 @@ def _insert(self, Model, name, price, quantity): def test_derived_dict_with_lambda(self, product_class): self._insert(product_class, "D1", 100.0, 4) results = product_class.find_all( - derived={"my_discount": lambda d: Column(d, "price") * Literal(d, 0.8)} + derived={ + "my_discount": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.8) + ) + } ) assert results[0].__dict__["my_discount"] == pytest.approx(80.0) @@ -153,7 +160,7 @@ def test_derived_dict_with_derived_field_instance(self, product_class): def test_derived_dict_with_expression_object(self, product_class): self._insert(product_class, "D3", 80.0, 5) dialect = product_class.backend().dialect - expr = Column(dialect, "price") * Literal(dialect, 0.5) + expr = BinaryArithmeticExpression(dialect, "*", Column(dialect, "price"), Literal(dialect, 0.5)) results = product_class.find_all( derived={"half_price": expr} ) @@ -185,7 +192,11 @@ def _insert(self, Model, name, price, quantity): def test_extra_derived_basic(self, product_class): self._insert(product_class, "F", 100.0, 10) results = product_class.find_all( - extra_derived={"triple_price": lambda d: Column(d, "price") * Literal(d, 3)} + extra_derived={ + "triple_price": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 3) + ) + } ) assert len(results) == 1 assert results[0].__dict__["triple_price"] == pytest.approx(300.0) @@ -194,14 +205,22 @@ def test_extra_derived_conflict_raises(self, product_class): self._insert(product_class, "G", 50.0, 2) with pytest.raises(ValueError, match="conflicts with a declared derived field"): product_class.find_all( - extra_derived={"discounted_price": lambda d: Column(d, "price") * Literal(d, 0.5)} + extra_derived={ + "discounted_price": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.5) + ) + } ) def test_derived_and_extra_derived_together(self, product_class): self._insert(product_class, "H", 40.0, 5) results = product_class.find_all( derived=True, - extra_derived={"double_qty": lambda d: Column(d, "quantity") * Literal(d, 2)} + extra_derived={ + "double_qty": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "quantity"), Literal(d, 2) + ) + } ) assert results[0].discounted_price == pytest.approx(36.0) assert results[0].total_value == pytest.approx(200.0) @@ -315,7 +334,9 @@ class ConflictModel(ActiveRecord): id: Optional[int] = None discount_rate: Annotated[float, UseColumn("disc")] discounted: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.9) + ), ), UseColumn("disc")]] def test_use_column_no_conflict_different_names(self): @@ -325,7 +346,9 @@ class NoConflict(ActiveRecord): id: Optional[int] = None discount_rate: Annotated[float, UseColumn("rate")] discounted: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.9) + ), ), UseColumn("disc")]] assert NoConflict.__derived_fields__["discounted"].column_name == "disc" diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field_async.py b/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field_async.py index 8720acd..d205397 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field_async.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fields/test_derived_field_async.py @@ -6,6 +6,7 @@ from rhosocial.activerecord.base import DerivedField, UseColumn from rhosocial.activerecord.model import ActiveRecord from rhosocial.activerecord.backend.expression import Column, Literal +from rhosocial.activerecord.backend.expression.operators import BinaryArithmeticExpression class TestAsyncDerivedFieldDeclaration: @@ -20,7 +21,9 @@ async def test_descriptor_class_access(self, async_product_class): async def test_derived_field_with_static_expression(self, async_product_class): """Test DerivedField constructed with a pre-built expression object (non-callable path).""" dialect = async_product_class.backend().dialect - static_expr = Column(dialect, "price") * Literal(dialect, 2) + static_expr = BinaryArithmeticExpression( + dialect, "*", Column(dialect, "price"), Literal(dialect, 2) + ) df = DerivedField(static_expr) resolved = df.resolve(dialect) assert resolved is static_expr @@ -137,7 +140,11 @@ async def _insert(self, Model, name, price, quantity): async def test_derived_dict_with_lambda(self, async_product_class): await self._insert(async_product_class, "D1", 100.0, 4) results = await async_product_class.find_all( - derived={"my_discount": lambda d: Column(d, "price") * Literal(d, 0.8)} + derived={ + "my_discount": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.8) + ) + } ) assert results[0].__dict__["my_discount"] == pytest.approx(80.0) @@ -153,7 +160,7 @@ async def test_derived_dict_with_derived_field_instance(self, async_product_clas async def test_derived_dict_with_expression_object(self, async_product_class): await self._insert(async_product_class, "D3", 80.0, 5) dialect = async_product_class.backend().dialect - expr = Column(dialect, "price") * Literal(dialect, 0.5) + expr = BinaryArithmeticExpression(dialect, "*", Column(dialect, "price"), Literal(dialect, 0.5)) results = await async_product_class.find_all( derived={"half_price": expr} ) @@ -185,7 +192,11 @@ async def _insert(self, Model, name, price, quantity): async def test_extra_derived_basic(self, async_product_class): await self._insert(async_product_class, "F", 100.0, 10) results = await async_product_class.find_all( - extra_derived={"triple_price": lambda d: Column(d, "price") * Literal(d, 3)} + extra_derived={ + "triple_price": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 3) + ) + } ) assert len(results) == 1 assert results[0].__dict__["triple_price"] == pytest.approx(300.0) @@ -194,14 +205,22 @@ async def test_extra_derived_conflict_raises(self, async_product_class): await self._insert(async_product_class, "G", 50.0, 2) with pytest.raises(ValueError, match="conflicts with a declared derived field"): await async_product_class.find_all( - extra_derived={"discounted_price": lambda d: Column(d, "price") * Literal(d, 0.5)} + extra_derived={ + "discounted_price": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.5) + ) + } ) async def test_derived_and_extra_derived_together(self, async_product_class): await self._insert(async_product_class, "H", 40.0, 5) results = await async_product_class.find_all( derived=True, - extra_derived={"double_qty": lambda d: Column(d, "quantity") * Literal(d, 2)} + extra_derived={ + "double_qty": lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "quantity"), Literal(d, 2) + ) + } ) assert results[0].discounted_price == pytest.approx(36.0) assert results[0].total_value == pytest.approx(200.0) @@ -315,7 +334,9 @@ class ConflictModel(ActiveRecord): id: Optional[int] = None discount_rate: Annotated[float, UseColumn("disc")] discounted: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.9) + ), ), UseColumn("disc")]] async def test_use_column_no_conflict_different_names(self): @@ -325,7 +346,9 @@ class NoConflict(ActiveRecord): id: Optional[int] = None discount_rate: Annotated[float, UseColumn("rate")] discounted: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Literal(d, 0.9) + ), ), UseColumn("disc")]] assert NoConflict.__derived_fields__["discounted"].column_name == "disc" diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fixtures/models.py b/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fixtures/models.py index ba5e9f0..93188d6 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fixtures/models.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/basic/fixtures/models.py @@ -767,7 +767,7 @@ class AsyncMappedOrderItem(CompositePKMixin, AsyncActiveRecord): # ============================================================================= from rhosocial.activerecord.base import DerivedField -from rhosocial.activerecord.backend.expression import Column, Literal +from rhosocial.activerecord.backend.expression import Column, Literal, BinaryArithmeticExpression class Product(ActiveRecord): @@ -779,11 +779,13 @@ class Product(ActiveRecord): quantity: int discounted_price: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression(d, "*", Column(d, "price"), Literal(d, 0.9)), )]] total_value: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Column(d, "quantity"), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Column(d, "quantity") + ), )]] @@ -796,11 +798,13 @@ class AsyncProduct(AsyncActiveRecord): quantity: int discounted_price: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression(d, "*", Column(d, "price"), Literal(d, 0.9)), )]] total_value: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Column(d, "quantity"), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Column(d, "quantity") + ), )]] @@ -814,11 +818,13 @@ class ProductFormA(ActiveRecord): quantity: int discounted_price: ClassVar[DerivedField] = DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression(d, "*", Column(d, "price"), Literal(d, 0.9)), ) total_value: ClassVar[DerivedField] = DerivedField( - lambda d: Column(d, "price") * Column(d, "quantity"), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Column(d, "quantity") + ), ) @@ -831,11 +837,13 @@ class AsyncProductFormA(AsyncActiveRecord): quantity: int discounted_price: ClassVar[DerivedField] = DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression(d, "*", Column(d, "price"), Literal(d, 0.9)), ) total_value: ClassVar[DerivedField] = DerivedField( - lambda d: Column(d, "price") * Column(d, "quantity"), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Column(d, "quantity") + ), ) @@ -900,11 +908,13 @@ class ProductWithColumnAndAdapter(ActiveRecord): quantity: int discounted_price: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression(d, "*", Column(d, "price"), Literal(d, 0.9)), ), UseColumn("disc")]] total_int: ClassVar[Annotated[int, DerivedField( - lambda d: Column(d, "price") * Column(d, "quantity"), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Column(d, "quantity") + ), ), UseAdapter(PriceToIntAdapter(), int)]] @@ -917,10 +927,12 @@ class AsyncProductWithColumnAndAdapter(AsyncActiveRecord): quantity: int discounted_price: ClassVar[Annotated[float, DerivedField( - lambda d: Column(d, "price") * Literal(d, 0.9), + lambda d: BinaryArithmeticExpression(d, "*", Column(d, "price"), Literal(d, 0.9)), ), UseColumn("disc")]] total_int: ClassVar[Annotated[int, DerivedField( - lambda d: Column(d, "price") * Column(d, "quantity"), + lambda d: BinaryArithmeticExpression( + d, "*", Column(d, "price"), Column(d, "quantity") + ), ), UseAdapter(PriceToIntAdapter(), int)]] diff --git a/tests/rhosocial/activerecord_clickhouse_test/feature/relation/fixtures/models.py b/tests/rhosocial/activerecord_clickhouse_test/feature/relation/fixtures/models.py index 1f02c19..2ae695e 100644 --- a/tests/rhosocial/activerecord_clickhouse_test/feature/relation/fixtures/models.py +++ b/tests/rhosocial/activerecord_clickhouse_test/feature/relation/fixtures/models.py @@ -14,6 +14,7 @@ from rhosocial.activerecord.model import ActiveRecord, AsyncActiveRecord from rhosocial.activerecord.base import DerivedField, FieldProxy from rhosocial.activerecord.backend.expression import Column, Literal +from rhosocial.activerecord.backend.expression.operators import BinaryArithmeticExpression from rhosocial.activerecord.backend.expression.functions import ( json_extract_text, length, concat, coalesce ) @@ -159,7 +160,7 @@ class Post(ActiveRecord): # DerivedField: hotness score hotness: ClassVar[Annotated[int, DerivedField( - lambda d: Column(d, "view_count") + Literal(d, 1), + lambda d: BinaryArithmeticExpression(d, "+", Column(d, "view_count"), Literal(d, 1)), )]] # DerivedField (JSON): first tag from metadata @@ -254,7 +255,7 @@ class AsyncPost(AsyncActiveRecord): )]] hotness: ClassVar[Annotated[int, DerivedField( - lambda d: Column(d, "view_count") + Literal(d, 1), + lambda d: BinaryArithmeticExpression(d, "+", Column(d, "view_count"), Literal(d, 1)), )]] first_tag: ClassVar[Annotated[str, DerivedField(