From 5b19437238ff585bf5d6529fbcd20dd15b0f6f59 Mon Sep 17 00:00:00 2001 From: Aliaksei Andryiashau Date: Mon, 12 Oct 2026 12:50:45 +0200 Subject: [PATCH 1/5] feat: collect and export a relations array per query Adds a per-query "relations" field: the distinct tables/views/materialized views/foreign tables/partitioned tables a query touches, schema-qualified, with the actual pg_class.relkind carried per entry. Collection: a new ExecutorCheckPerms_hook walks the resolved range table, dedupes same-relation-twice, and resolves names via get_namespace_name/get_rel_name. Capped by pg_stat_ch.relations_max_len. Reentrancy: each nested executor invocation gets its own staging buffer, keyed to its QueryDesc rather than call depth. Transaction/subtransaction abort and commit callbacks keep the table consistent across aborts, commits, and reused nesting levels. A cursor fetched from inside a savepoint that then fails is tracked by use-site as well as create-site, so PostgreSQL failing that cursor mid-abort (which skips our usual cleanup) can't permanently strand its reservation. Gating: the relation_namespaces/relation_names/relation_kinds/ relations_truncated columns are only declared to the exporter when pg_stat_ch.relations_enabled is on, so an unmigrated ClickHouse schema still accepts inserts. The GUC is PGC_POSTMASTER, since the ClickHouse exporter keeps its column set for the life of the bgworker. When disabled, the hook also skips writing the empty-relations header, so there's no wasted per-query DSA allocation. Storage/export: rides through the existing ring buffer, same as query text and error messages. Exported as real array types on ClickHouse (Array(LowCardinality(String)) / Array(FixedString(1))), OTLP (array-value log attribute), and the unified Arrow exporter (list(utf8)). The legacy Arrow exporter doesn't carry relations yet. Schema: a single migration adds the four relation columns (ADD COLUMN IF NOT EXISTS), leaving events_recent_1h untouched -- same convention as the labels migration, since ClickHouse won't let the view's frozen SELECT * pick up new columns without a drop/recreate. Non-UTF-8 relation names are a known, pre-existing gap (same one already documented for query text and error messages) -- left for a follow-up PR. Testing: regression, TAP, and unit coverage for the GUC, relation shapes, nested-invocation handling, failure recovery, the cursor/savepoint reservation leak, and the pure-function pieces (budget check, encoding, nesting push/pop/abort/reparent). The unified Arrow exporter's relation columns are covered end-to-end across two separately-flushed batches. --- .github/workflows/ci.yml | 5 +- CMakeLists.txt | 22 + README.md | 2 + docker/postgres-ext.Dockerfile | 1 + docs/guides/opentelemetry.mdx | 1 + docs/reference/configuration.mdx | 41 + docs/reference/events-schema.mdx | 13 + docs/reference/hooks.mdx | 18 + include/config/guc.h | 2 + include/hooks/relations_budget.h | 30 + include/hooks/relations_nesting.h | 130 +++ .../20261116000001_add_relation_columns.sql | 20 + scripts/run-tests.sh | 2 +- src/config/guc.c | 32 + src/export/clickhouse_exporter.cc | 38 + src/export/exporter_interface.h | 18 + src/export/otel_arrow_exporter.cc | 48 ++ src/export/otel_exporter.cc | 24 + src/export/otel_string_array_attr.h | 43 + src/export/relations_decode.h | 75 ++ src/export/stats_exporter.cc | 21 + src/hooks/hooks.c | 260 +++++- src/queue/event.h | 18 +- src/queue/ring_entry.h | 19 +- src/queue/shmem.c | 32 +- t/036_unified_arrow_e2e.pl | 60 +- t/039_relations_export.pl | 785 ++++++++++++++++++ test/regression/expected/guc.out | 4 +- test/regression/expected/relations.out | 117 +++ test/regression/pg_stat_ch.conf | 1 + test/regression/sql/relations.sql | 67 ++ test/unit/otel_string_array_attr_test.cc | 61 ++ test/unit/relations_budget_test.cc | 19 + test/unit/relations_decode_test.cc | 113 +++ test/unit/relations_nesting_test.cc | 229 +++++ vcpkg.json | 3 +- 36 files changed, 2336 insertions(+), 38 deletions(-) create mode 100644 include/hooks/relations_budget.h create mode 100644 include/hooks/relations_nesting.h create mode 100644 schema/migrations/20261116000001_add_relation_columns.sql create mode 100644 src/export/otel_string_array_attr.h create mode 100644 src/export/relations_decode.h create mode 100644 t/039_relations_export.pl create mode 100644 test/regression/expected/relations.out create mode 100644 test/regression/sql/relations.sql create mode 100644 test/unit/otel_string_array_attr_test.cc create mode 100644 test/unit/relations_budget_test.cc create mode 100644 test/unit/relations_decode_test.cc create mode 100644 test/unit/relations_nesting_test.cc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ac0b96..db99642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,9 @@ jobs: - name: Build run: cmake --build build --parallel + - name: Run unit tests + run: ctest --test-dir build --output-on-failure + - name: Install extension run: sudo cmake --install build @@ -83,7 +86,7 @@ jobs: --outputdir=test/regression/results \ --temp-instance=test/regression/tmp_check \ --temp-config=test/regression/pg_stat_ch.conf \ - basic version guc stats utility buffers cmd_type client_info error_capture upgrade + basic version guc stats utility buffers cmd_type client_info error_capture upgrade relations - name: Run isolation tests run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 157f7c0..513d0af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,3 +146,25 @@ install(FILES pg_stat_ch.control DESTINATION ${PostgreSQLServer_SHARE_DIR}/exten file(GLOB SQL_FILES sql/pg_stat_ch--*.sql) install(FILES ${SQL_FILES} DESTINATION ${PostgreSQLServer_SHARE_DIR}/extension) +# Unit tests for plain C++ helpers that don't need a running Postgres (see +# test/unit/). Separate from the pg_regress/TAP suites, which do. Off by +# default for image builds that COPY a .dockerignore-filtered context +# without test/ -- those only need the pg_stat_ch target, never this one. +option(PSCH_BUILD_UNIT_TESTS "Build the GoogleTest C++ unit tests" ON) +if(PSCH_BUILD_UNIT_TESTS) + enable_testing() + find_package(GTest CONFIG REQUIRED) + add_executable(pg_stat_ch_unittests + test/unit/relations_decode_test.cc + test/unit/relations_budget_test.cc + test/unit/relations_nesting_test.cc + test/unit/otel_string_array_attr_test.cc + ) + target_compile_features(pg_stat_ch_unittests PRIVATE cxx_std_17) + target_include_directories(pg_stat_ch_unittests PRIVATE src include) + target_link_libraries(pg_stat_ch_unittests PRIVATE GTest::gtest GTest::gtest_main + opentelemetry-cpp::otlp_grpc_log_record_exporter) + include(GoogleTest) + gtest_discover_tests(pg_stat_ch_unittests) +endif() + diff --git a/README.md b/README.md index 492d627..9c27d4c 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ SELECT * FROM pg_stat_ch_stats(); | `pg_stat_ch.clickhouse_use_tls` | bool | `off` | Restart | Enable TLS for ClickHouse connections | | `pg_stat_ch.clickhouse_skip_tls_verify` | bool | `off` | Restart | Skip TLS certificate verification (insecure) | | `pg_stat_ch.log_min_elevel` | enum | `warning` | Superuser | Minimum error level to capture (debug5..panic) | +| `pg_stat_ch.relations_enabled` | bool | `off` | Restart | Collect the relations array (tables/views a query touches); apply the relations schema migrations before enabling | +| `pg_stat_ch.relations_max_len` | int | `1023` | Superuser | Maximum encoded size in bytes of the per-query relations blob | See [Error Level Values](docs/version-compatibility.md#error-fields) for the complete list of error levels and their numeric values in ClickHouse. diff --git a/docker/postgres-ext.Dockerfile b/docker/postgres-ext.Dockerfile index ada94f9..a1e2cda 100644 --- a/docker/postgres-ext.Dockerfile +++ b/docker/postgres-ext.Dockerfile @@ -61,6 +61,7 @@ COPY third_party/clickhouse-c/ third_party/clickhouse-c/ RUN cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DPSCH_BUILD_UNIT_TESTS=OFF \ -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ -DVCPKG_TARGET_TRIPLET="$(cat /build/vcpkg-triplet)" \ -DVCPKG_HOST_TRIPLET="$(cat /build/vcpkg-triplet)" \ diff --git a/docs/guides/opentelemetry.mdx b/docs/guides/opentelemetry.mdx index da4c27b..f82c661 100644 --- a/docs/guides/opentelemetry.mdx +++ b/docs/guides/opentelemetry.mdx @@ -21,6 +21,7 @@ When `use_otel` is enabled, the ClickHouse connection parameters are ignored. Th The OTel exporter maps pg_stat_ch events to OpenTelemetry semantic conventions: - **Logs**: Each query execution becomes an OTel log record with attributes following the [database semantic conventions](https://opentelemetry.io/docs/specs/semconv/database/) (`db.name`, `db.user`, `db.operation.name`, `db.query.text`). +- **Relations**: tables/views touched by the query are exported as 3 separate native OTel array attributes, positionally parallel to each other: `relation_namespaces` (schema names), `relation_names` (relation names), `relation_kinds` (single-character `pg_class.relkind` per entry: `r`=table, `v`=view, `m`=matview, `f`=foreign table, `p`=partitioned table, ...). This mirrors the 3-array shape used for the ClickHouse-native export path; neither shape is final -- OTLP 1.9.0 added complex-type array attributes, and the ClickHouse schema may move to `Array(Tuple(...))`, so this representation may change. The exporter builds OTLP log requests directly in the bgworker. pg_stat_ch's shared-memory queue already buffers events, and the exporter chunks those events into bounded gRPC requests. diff --git a/docs/reference/configuration.mdx b/docs/reference/configuration.mdx index 680adfe..24c3f00 100644 --- a/docs/reference/configuration.mdx +++ b/docs/reference/configuration.mdx @@ -141,6 +141,47 @@ Extract [sqlcommenter](https://google.github.io/sqlcommenter/) labels from query When enabled, the background worker parses the `/* key='value' */` comment that ends each query and exports the pairs as a JSON object in the `labels` column (the `labels` attribute in OpenTelemetry). When disabled, no `labels` column is sent, so ClickHouse stores its `{}` default. Apply the `labels` schema migration before enabling it. +## Relations + +### `pg_stat_ch.relations_enabled` + +Collect the relations array (tables/views a query touches). + +| | | +|---|---| +| **Type** | boolean | +| **Default** | `off` | +| **Context** | postmaster | + +When enabled, pg_stat_ch resolves each query's touched relations (tables, views, materialized views, foreign tables, partitioned tables) and exports them as three parallel arrays -- `relation_namespaces`/`relation_names`/`relation_kinds` -- plus a `relations_truncated` flag. Toggling this off skips the catalog lookups entirely, not just the storage -- useful for isolating this feature's hot-path cost from pg_stat_ch's baseline overhead. In a synthetic benchmark across join widths of 1/5/10/20 tables, the added CPU cost measured between +0.05us and +1.75us per query. + +PGC_POSTMASTER: the ClickHouse exporter keeps its column set for the life of the bgworker, so changing this requires a restart. Apply the relations schema migrations before enabling it. + +```sql +-- Enable relations collection after applying the schema migrations +ALTER SYSTEM SET pg_stat_ch.relations_enabled = on; +-- then restart PostgreSQL +``` + +### `pg_stat_ch.relations_max_len` + +Maximum encoded size, in bytes, of the per-query relations blob. + +| | | +|---|---| +| **Type** | integer | +| **Default** | `1023` | +| **Min** | `2` (just enough room for the 2-byte header) | +| **Max** | `1023` | +| **Context** | suset | + +Queries touching enough distinct relations to exceed this budget are truncated: `relations_truncated` is set to `true` for that event, and the relation arrays contain only as many entries as fit. Lower this if you want to bound the per-event ClickHouse payload size further; it cannot be raised above the compiled-in maximum. + +```sql +-- Lower the budget, e.g. to bound payload size more aggressively +SET pg_stat_ch.relations_max_len = 256; +``` + ## Queue The shared-memory ring buffer sits between the foreground hooks and the background worker. These settings control its size and drain behavior. diff --git a/docs/reference/events-schema.mdx b/docs/reference/events-schema.mdx index dee5413..d78182e 100644 --- a/docs/reference/events-schema.mdx +++ b/docs/reference/events-schema.mdx @@ -44,6 +44,19 @@ Query text is cut at 2,047 bytes, and text that reaches that length gets no labe Label values are exported as written by the application. Don't put PII, tokens, or secrets in sqlcommenter comments. +## Relations + +| Column | Type | Description | +|---|---|---| +| `relation_namespaces` | `Array(LowCardinality(String))` | Schema name of each distinct table/view/materialized view/foreign table/partitioned table the query touched, positionally parallel to `relation_names` and `relation_kinds`. | +| `relation_names` | `Array(LowCardinality(String))` | Relation name of each distinct relation the query touched. | +| `relation_kinds` | `Array(FixedString(1))` | `pg_class.relkind` for each entry: `r` (table), `v` (view), `m` (materialized view), `f` (foreign table), `p` (partitioned table), etc. | +| `relations_truncated` | `Bool` | `true` if the query touched more relations than fit within `pg_stat_ch.relations_max_len`; the arrays above are incomplete for that event. | + + +These columns require [`pg_stat_ch.relations_enabled`](/reference/configuration#pg_stat_chrelations_enabled) (off by default -- apply the relations schema migrations, then enable it). Once enabled, all three relation arrays are still empty for utility statements and for queries that touch no relations. See [`pg_stat_ch.relations_max_len`](/reference/configuration#pg_stat_chrelations_max_len) for the truncation budget. `events_recent_1h` keeps the columns it was created with, so it has no relation columns. + + ## Shared buffer usage These columns show how queries interact with PostgreSQL's `shared_buffers` cache. The cache hit ratio for a query is `shared_blks_hit / (shared_blks_hit + shared_blks_read)`. Target above 99% for OLTP workloads. diff --git a/docs/reference/hooks.mdx b/docs/reference/hooks.mdx index 18e23e4..9297291 100644 --- a/docs/reference/hooks.mdx +++ b/docs/reference/hooks.mdx @@ -10,6 +10,7 @@ pg_stat_ch uses PostgreSQL's extensibility hooks to capture query telemetry with | Hook | Purpose | When Called | |------|---------|-------------| | `ExecutorStart_hook` | Initialize instrumentation | Before query execution begins | +| `ExecutorCheckPerms_hook` | Collect touched relations | During permission checking, early in executor startup | | `ExecutorRun_hook` | Track nesting level | During query execution | | `ExecutorFinish_hook` | Track nesting level | After execution, before cleanup | | `ExecutorEnd_hook` | Capture metrics and enqueue event | After query completes | @@ -32,6 +33,20 @@ The executor hooks work together to capture metrics for optimizable queries (SEL 4. Initializes CPU time baseline via `getrusage()` 5. Enables instrumentation to collect buffer/timing stats +### ExecutorCheckPerms_hook + +**Called:** During permission checking, as part of `ExecutorStart` -- PostgreSQL hands every relation in the query's range table to this hook before deciding whether the current role may access it. + +**What pg_stat_ch does:** + +1. Skips parallel workers (the leader already captures this) +2. Walks the range table, resolving each distinct relation to its schema-qualified name and `pg_class.relkind` +3. Encodes the result into a length-prefixed binary buffer, bounded by `pg_stat_ch.relations_max_len`; queries that exceed the budget are truncated and flagged via `relations_truncated` +4. Stages the buffer in a per-nesting-level slot rather than a single shared buffer, so a nested executor invocation (a non-inlined SQL function, PL/pgSQL, dynamic `EXECUTE`, a trigger, SPI) can't clobber the outer statement's already-collected relations +5. Defers to the previous hook (or allows by default) for the actual permission decision -- this hook only observes, it never denies access + +Collection is skipped entirely (not just discarded afterward) when `pg_stat_ch.relations_enabled` is off. + ### ExecutorRun_hook **Called:** When the executor actually runs the query plan. @@ -175,6 +190,9 @@ Parallel workers execute portions of a query plan. pg_stat_ch: │ ├─ Record start time, CPU baseline │ │ └─ Enable instrumentation │ │ │ +│ ExecutorCheckPerms_hook │ +│ └─ Collect touched relations (schema, name, relkind) │ +│ │ │ ExecutorRun_hook / ExecutorFinish_hook │ │ └─ Track nesting level │ │ │ diff --git a/include/config/guc.h b/include/config/guc.h index 75e1066..efe574c 100644 --- a/include/config/guc.h +++ b/include/config/guc.h @@ -38,6 +38,8 @@ extern bool psch_use_unified_arrow_exporter; extern int psch_otel_max_block_bytes; extern char* psch_extra_attributes; extern char* psch_debug_arrow_dump_dir; +extern int psch_relations_max_len; +extern bool psch_relations_enabled; // Initialize GUC variables void PschInitGuc(void); diff --git a/include/hooks/relations_budget.h b/include/hooks/relations_budget.h new file mode 100644 index 0000000..29000e9 --- /dev/null +++ b/include/hooks/relations_budget.h @@ -0,0 +1,30 @@ +// Pure C, no PostgreSQL headers -- kept separate from hooks.c so it's +// unit-testable in isolation (see test/unit/relations_budget_test.cc). +#ifndef PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ +#define PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// [truncated:u8][count:u8] -- the GUC's minimum must never go below this. +#define PSCH_RELATIONS_HEADER_LEN 2 + +// Sentinel relations_len for "blob lost to DSA OOM" -- distinct from 0 (no +// relations) and real blob lengths (>= PSCH_RELATIONS_HEADER_LEN). +#define PSCH_RELATIONS_OOM_LEN 1 + +// True if appending an entry of entry_len bytes to a buffer that already +// holds current_len bytes would exceed budget. No separator byte to +// account for -- every entry is self-delimiting (length-prefixed). +static inline bool PschRelationsWouldExceedBudget(int current_len, int entry_len, int budget) { + return (current_len + entry_len) > budget; +} + +#ifdef __cplusplus +} +#endif + +#endif // PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ diff --git a/include/hooks/relations_nesting.h b/include/hooks/relations_nesting.h new file mode 100644 index 0000000..0314758 --- /dev/null +++ b/include/hooks/relations_nesting.h @@ -0,0 +1,130 @@ +// Pure C, no PostgreSQL headers -- kept separate from hooks.c so it's +// unit-testable in isolation (see test/unit/relations_nesting_test.cc). +#ifndef PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_NESTING_H_ +#define PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_NESTING_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Keyed by identity (e.g. a QueryDesc*), not by call depth -- so a level +// can be released out of order (concurrently open cursors don't close in +// the order they were opened) without handing one query's buffer to +// another. `created_level` and `active_level` (deepest level used at, see +// PschRelationsNestingTouch) are opaque caller-defined tags used by +// PschRelationsNestingAbortTo. +typedef struct { + const void* key; + int created_level; + int active_level; + bool in_use; +} PschRelationsNestingEntry; + +// Returns the index of the in-use slot reserved for `key`, or -1 if none. +static inline int PschRelationsNestingFind(const PschRelationsNestingEntry* table, int max_depth, + const void* key) { + for (int i = 0; i < max_depth; i++) { + if (table[i].in_use && table[i].key == key) { + return i; + } + } + return -1; +} + +// Reserves a slot for `key`, returning its index, or -1 if all max_depth +// slots are already in use. Reclaims an existing in-use entry for the same +// key instead of adding a duplicate -- guards against a prior reservation +// for this address never having been released (see PschRelationsNestingPop). +static inline int PschRelationsNestingPush(PschRelationsNestingEntry* table, int max_depth, + const void* key, int created_level) { + int reclaim = PschRelationsNestingFind(table, max_depth, key); + if (reclaim >= 0) { + table[reclaim].created_level = created_level; + table[reclaim].active_level = created_level; + return reclaim; + } + for (int i = 0; i < max_depth; i++) { + if (!table[i].in_use) { + table[i].key = key; + table[i].created_level = created_level; + table[i].active_level = created_level; + table[i].in_use = true; + return i; + } + } + return -1; +} + +// Records `key` as used at `level`, returning its prior active_level (or -1 +// if unknown). Never lowers active_level; call SetActiveLevel to restore it. +static inline int PschRelationsNestingTouch(PschRelationsNestingEntry* table, int max_depth, + const void* key, int level) { + int i = PschRelationsNestingFind(table, max_depth, key); + if (i < 0) { + return -1; + } + int prev = table[i].active_level; + if (level > prev) { + table[i].active_level = level; + } + return prev; +} + +// Restores active_level to `level`; no-op if unknown. +static inline void PschRelationsNestingSetActiveLevel(PschRelationsNestingEntry* table, + int max_depth, const void* key, int level) { + int i = PschRelationsNestingFind(table, max_depth, key); + if (i >= 0) { + table[i].active_level = level; + } +} + +// Frees and returns the slot reserved for `key`, or -1 if none is (e.g. Push +// returned -1 for it, or PschRelationsNestingAbortTo already released it). +static inline int PschRelationsNestingPop(PschRelationsNestingEntry* table, int max_depth, + const void* key) { + int i = PschRelationsNestingFind(table, max_depth, key); + if (i >= 0) { + table[i].in_use = false; + } + return i; +} + +// Frees every slot created or used at `level` or deeper -- covers both +// rolled-back pushes and older portals PostgreSQL fails mid-abort. +static inline void PschRelationsNestingAbortTo(PschRelationsNestingEntry* table, int max_depth, + int level) { + for (int i = 0; i < max_depth; i++) { + if (table[i].in_use && + (table[i].created_level >= level || table[i].active_level >= level)) { + table[i].in_use = false; + } + } +} + +// A committing (sub)transaction's surviving entries (e.g. a cursor opened +// inside it) must move to its parent's level -- otherwise a later sibling +// subtransaction reusing this same level number would wrongly sweep them on +// its own abort. +static inline void PschRelationsNestingReparentCommitted(PschRelationsNestingEntry* table, + int max_depth, int committed_level) { + for (int i = 0; i < max_depth; i++) { + if (!table[i].in_use) { + continue; + } + if (table[i].created_level == committed_level) { + table[i].created_level = committed_level - 1; + } + if (table[i].active_level == committed_level) { + table[i].active_level = committed_level - 1; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif // PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_NESTING_H_ diff --git a/schema/migrations/20261116000001_add_relation_columns.sql b/schema/migrations/20261116000001_add_relation_columns.sql new file mode 100644 index 0000000..6393409 --- /dev/null +++ b/schema/migrations/20261116000001_add_relation_columns.sql @@ -0,0 +1,20 @@ +-- Adds structured relation columns: namespace, name, and relkind kept as +-- separate parallel arrays. Collected in src/hooks/hooks.c +-- (ExecutorCheckPerms_hook). + +-- +goose Up + +-- +goose StatementBegin +ALTER TABLE pg_stat_ch.events_raw + ADD COLUMN IF NOT EXISTS relation_namespaces Array(LowCardinality(String)) COMMENT 'Schema name per entry, positionally parallel to relation_names/relation_kinds.', + ADD COLUMN IF NOT EXISTS relation_names Array(LowCardinality(String)) COMMENT 'Relation name per entry, positionally parallel to relation_namespaces/relation_kinds.', + ADD COLUMN IF NOT EXISTS relation_kinds Array(FixedString(1)) COMMENT 'pg_class.relkind char per entry (r=table, v=view, m=matview, f=foreign table, p=partitioned table, ...).', + ADD COLUMN IF NOT EXISTS relations_truncated Bool DEFAULT false COMMENT 'True if pg_stat_ch.relations_max_len was exceeded and this list is incomplete.'; +-- +goose StatementEnd + +-- +goose Down + +-- Keep the columns on rollback: ClickHouse refuses to drop them while +-- events_recent_1h's SELECT * depends on them, and producers that don't send +-- them still insert. +SELECT 1; diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 98033bd..9c8ed2a 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -110,7 +110,7 @@ run_regress() { --outputdir=test/regression/results \ --temp-instance=test/regression/tmp_check \ --temp-config=test/regression/pg_stat_ch.conf \ - basic version guc stats utility buffers cmd_type client_info error_capture drop_database_barrier upgrade + basic version guc stats utility buffers cmd_type client_info error_capture drop_database_barrier upgrade relations } # Run TAP tests diff --git a/src/config/guc.c b/src/config/guc.c index d6a80ff..eeb1bb9 100644 --- a/src/config/guc.c +++ b/src/config/guc.c @@ -7,6 +7,7 @@ #include #include "config/guc.h" +#include "hooks/relations_budget.h" // GUC variable storage bool psch_enabled = true; @@ -40,6 +41,8 @@ bool psch_use_unified_arrow_exporter = false; int psch_otel_max_block_bytes = 3 * 1024 * 1024; // 3 MiB (max: 16 MiB) char* psch_extra_attributes = NULL; char* psch_debug_arrow_dump_dir = NULL; +int psch_relations_max_len = 1023; // bytes, leaves room for the NUL in PSCH_MAX_RELATIONS_LEN=1024 +bool psch_relations_enabled = false; // Log level options (matches PostgreSQL's server_message_level_options pattern) // clang-format off @@ -410,6 +413,35 @@ void PschInitGuc(void) { 0, NULL, NULL, NULL); + DefineCustomIntVariable( + "pg_stat_ch.relations_max_len", + "Maximum bytes for the encoded list of tables/views a query touches.", + "Mirrors the query-text truncation pattern: the relations blob is a fixed " + "backend-local buffer, truncated (not grown) when a query touches enough " + "distinct relations to exceed this budget.", + &psch_relations_max_len, + 1023, // bootValue + PSCH_RELATIONS_HEADER_LEN, // min -- the header alone must still fit + 1023, // max (PSCH_MAX_RELATIONS_LEN - 1, room for the NUL) + PGC_SUSET, + 0, + NULL, NULL, NULL); + + DefineCustomBoolVariable( + "pg_stat_ch.relations_enabled", + "Collect the relations array (tables/views a query touches).", + "When enabled, the background worker resolves each query's touched " + "relations via ExecutorCheckPerms_hook and exports them as the " + "relation_namespaces/relation_names/relation_kinds/relations_truncated " + "columns. Apply the relations schema migrations before enabling. " + "PGC_POSTMASTER: the ClickHouse exporter keeps its column set for the " + "life of the bgworker, so a runtime change would leave a stale column.", + &psch_relations_enabled, + false, + PGC_POSTMASTER, + 0, + NULL, NULL, NULL); + DefineCustomBoolVariable( "pg_stat_ch.track_labels", "Extract sqlcommenter labels from query comments.", diff --git a/src/export/clickhouse_exporter.cc b/src/export/clickhouse_exporter.cc index 19c6a41..09fc7e3 100644 --- a/src/export/clickhouse_exporter.cc +++ b/src/export/clickhouse_exporter.cc @@ -180,6 +180,9 @@ class ClickHouseExporter : public StatsExporter { shared_ptr> StatHCUInt64(string_view name) final { return MakeCol>(name, "UInt64"); } + shared_ptr>> StatHCStringArray(string_view name) final { + return MakeCol(name); + } shared_ptr> StatTimestamp(string_view name) final { return MakeCol>(name, "DateTime64(6)"); @@ -263,6 +266,41 @@ class ClickHouseExporter : public StatsExporter { chc_column col_{}; }; + // Array(String) column: same flattened-bytes approach as StringCol, plus + // row_offsets_ tracking each row's cumulative element count. + class ArrayStringCol : public Column>, public ChColumn { + public: + ArrayStringCol(ClickHouseExporter* exp, string_view name) : exp_(exp), name_(name) {} + void Append(const std::vector& arr) final { + for (const auto& s : arr) { + bytes_.insert(bytes_.end(), s.data(), s.data() + s.size()); + value_offsets_.push_back(bytes_.size()); + } + row_offsets_.push_back(value_offsets_.size()); + } + bool Crunch() final { + values_col_ = chc_build_string(value_offsets_.data(), + reinterpret_cast(bytes_.data()), + value_offsets_.size()); + col_ = chc_build_array(row_offsets_.data(), row_offsets_.size(), &values_col_); + return exp_->AppendColumn(name_, "Array(String)", &col_); + } + void Clear() final { + bytes_.clear(); + value_offsets_.clear(); + row_offsets_.clear(); + } + + private: + ClickHouseExporter* const exp_; + const std::string name_; + std::vector bytes_; + std::vector value_offsets_; + std::vector row_offsets_; + chc_column values_col_{}; + chc_column col_{}; + }; + template shared_ptr MakeCol(string_view name, Args&&... args) { if (auto it = col_index_.find(name); it != col_index_.end()) { diff --git a/src/export/exporter_interface.h b/src/export/exporter_interface.h index a70abfb..bef6a15 100644 --- a/src/export/exporter_interface.h +++ b/src/export/exporter_interface.h @@ -6,6 +6,7 @@ #include #include #include +#include class StatsExporter { protected: @@ -21,6 +22,15 @@ class StatsExporter { virtual ~Column() = default; }; + // No-op column: the default StatHCStringArray() implementation below + // returns this so callers never need a null check, matching the "not every + // backend implements every column" pattern SendArrowBatch already uses. + template + class NoopColumn : public Column { + public: + void Append(const T& /*t*/) final {} + }; + public: // =========================================================================== // Cardinality-typed columns. @@ -55,6 +65,14 @@ class StatsExporter { virtual shared_ptr> StatHCInt64(string_view name) = 0; virtual shared_ptr> StatHCUInt64(string_view name) = 0; + // Array-of-strings column. Non-pure with a no-op default so a backend + // that doesn't implement it yet simply discards, rather than a pure + // virtual forcing every backend to handle it explicitly. + virtual shared_ptr>> StatHCStringArray(string_view name) { + (void)name; + return std::make_shared>>(); + } + // Domain-specific. Caller appends a Unix-epoch microsecond timestamp. // (PG-epoch values must be offset by kPostgresEpochOffsetUs before append; // CH DateTime64(6) and OTel time_unix_nano both interpret the wire value diff --git a/src/export/otel_arrow_exporter.cc b/src/export/otel_arrow_exporter.cc index ad85c91..0ccaf71 100644 --- a/src/export/otel_arrow_exporter.cc +++ b/src/export/otel_arrow_exporter.cc @@ -21,6 +21,7 @@ extern "C" { #include #include +#include #include #include #include @@ -180,6 +181,9 @@ class OTelArrowExporter : public StatsExporter { shared_ptr> StatHCUInt64(string_view n) final { return MakeNum(n, arrow::uint64()); } + shared_ptr>> StatHCStringArray(string_view n) final { + return MakeListUtf8Sv(n); + } shared_ptr> StatTimestamp(string_view n) final { return MakeTimestamp(n); } @@ -301,6 +305,39 @@ class OTelArrowExporter : public StatsExporter { size_t* const bytes_; }; + // list(utf8) column for StatHCStringArray -- mirrors the OTelExporter's + // array_value log attribute (used for relation_namespaces/relation_names/ + // relation_kinds, e.g. relation_names -> ["foo1","foo3"], namespace kept + // in the parallel relation_namespaces array, not inlined here), just + // built as an Arrow list instead of an OTLP AnyValue.array_value. + class ArrowListUtf8SvColumn : public Column> { + public: + ArrowListUtf8SvColumn(arrow::ListBuilder* builder, arrow::StringBuilder* value_builder, + size_t* bytes) + : builder_(builder), value_builder_(value_builder), bytes_(bytes) {} + void Append(const std::vector& v) final { + arrow::Status s = builder_->Append(); + if (!s.ok()) { + LogArrowFailure("Arrow list append", s); + return; + } + *bytes_ += kVarLenOffsetBytes; + for (const auto& elem : v) { + s = value_builder_->Append(elem.data(), static_cast(elem.size())); + if (!s.ok()) { + LogArrowFailure("Arrow list element append", s); + return; + } + *bytes_ += elem.size() + kVarLenOffsetBytes; + } + } + + private: + arrow::ListBuilder* const builder_; + arrow::StringBuilder* const value_builder_; + size_t* const bytes_; + }; + class ArrowTimestampColumn : public Column { public: ArrowTimestampColumn(arrow::TimestampBuilder* builder, size_t* bytes) @@ -351,6 +388,17 @@ class OTelArrowExporter : public StatsExporter { {std::string(name), arrow::field(std::string(name), arrow::utf8()), std::move(builder)}); return std::make_shared(raw, &bytes_estimate_); } + shared_ptr>> MakeListUtf8Sv(string_view name) { + auto value_builder = std::make_shared(); + auto* raw_value = value_builder.get(); + auto builder = + std::make_shared(arrow::default_memory_pool(), value_builder); + auto* raw = builder.get(); + slots_.push_back({std::string(name), + arrow::field(std::string(name), arrow::list(arrow::utf8())), + std::move(builder)}); + return std::make_shared(raw, raw_value, &bytes_estimate_); + } shared_ptr> MakeTimestamp(string_view name) { auto builder = std::make_shared(TimestampType(), arrow::default_memory_pool()); diff --git a/src/export/otel_exporter.cc b/src/export/otel_exporter.cc index a97ea8b..fcb6560 100644 --- a/src/export/otel_exporter.cc +++ b/src/export/otel_exporter.cc @@ -17,6 +17,7 @@ #include "config/guc.h" #include "export/exporter_interface.h" +#include "export/otel_string_array_attr.h" #include #include @@ -181,6 +182,9 @@ class OTelExporter : public StatsExporter { shared_ptr> StatHCUInt64(string_view name) final { return MakeIntCol(name); } + shared_ptr>> StatHCStringArray(string_view name) final { + return MakeArrayStrCol(name); + } shared_ptr> StatTimestamp(string_view name) final { return MakeDateTimeCol(name); @@ -239,6 +243,22 @@ class OTelExporter : public StatsExporter { string name_; }; + class ArrayStrColumn : public Column> { + public: + ArrayStrColumn(OTelExporter* e, string_view n) : exp_(e), name_(n) {} + void Append(const std::vector& v) final { + if (exp_->current_record_ == nullptr) { + return; + } + psch::SetStringArrayAttr(AddAttr(exp_->current_record_), name_, v); + exp_->chunk_bytes_ += psch::EstimateStringArrayAttrBytes(name_, v); + } + + private: + OTelExporter* exp_; + string name_; + }; + class StrColumn : public Column { public: StrColumn(OTelExporter* e, string_view n) : exp_(e), name_(n) {} @@ -301,6 +321,10 @@ class OTelExporter : public StatsExporter { return std::make_shared(this, name); } + shared_ptr>> MakeArrayStrCol(string_view name) { + return std::make_shared(this, name); + } + shared_ptr> MakeStringCol(string_view name) { return std::make_shared(this, name); } diff --git a/src/export/otel_string_array_attr.h b/src/export/otel_string_array_attr.h new file mode 100644 index 0000000..0c572d5 --- /dev/null +++ b/src/export/otel_string_array_attr.h @@ -0,0 +1,43 @@ +// Builds an array-valued OTLP attribute for any HC string-array stat. +// Separate from otel_exporter.cc so it's unit-testable without a live +// gRPC channel (test/unit/otel_string_array_attr_test.cc). +#ifndef PG_STAT_CH_SRC_EXPORT_OTEL_STRING_ARRAY_ATTR_H_ +#define PG_STAT_CH_SRC_EXPORT_OTEL_STRING_ARRAY_ATTR_H_ + +#include + +#include +#include +#include + +namespace psch { + +// AnyValue.array_value is a plain repeated AnyValue -- every element gets +// its own string_value, same as a top-level string attribute would. +inline void SetStringArrayAttr(opentelemetry::proto::common::v1::KeyValue* kv, + std::string_view key, + const std::vector& values) { + kv->set_key(key.data(), key.size()); + auto* arr = kv->mutable_value()->mutable_array_value(); + for (const auto& value : values) { + arr->add_values()->set_string_value(value.data(), value.size()); + } +} + +// Used to decide when a chunk is full. Unlike a scalar attribute, each +// array element is its own nested AnyValue, with its own protobuf framing +// on top of the string bytes -- kPerElementOverheadBytes covers that. +inline size_t EstimateStringArrayAttrBytes(std::string_view key, + const std::vector& values) { + constexpr size_t kAttrOverheadBytes = 24; + constexpr size_t kPerElementOverheadBytes = 4; + size_t total = kAttrOverheadBytes + key.size(); + for (const auto& value : values) { + total += value.size() + kPerElementOverheadBytes; + } + return total; +} + +} // namespace psch + +#endif // PG_STAT_CH_SRC_EXPORT_OTEL_STRING_ARRAY_ATTR_H_ diff --git a/src/export/relations_decode.h b/src/export/relations_decode.h new file mode 100644 index 0000000..b79d1df --- /dev/null +++ b/src/export/relations_decode.h @@ -0,0 +1,75 @@ +// Pure C++, no PostgreSQL headers -- kept separate from stats_exporter.cc so +// it's unit-testable in isolation (see test/unit/relations_decode_test.cc). +#ifndef PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ +#define PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ + +#include +#include +#include + +#include "hooks/relations_budget.h" + +// Decoded form of the relations blob built by CollectRelations in hooks.c: +// [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count +struct DecodedRelations { + bool truncated = false; + std::vector namespaces; + std::vector names; + std::vector kinds; // one-byte spans, e.g. "r", "v", "m" +}; + +// Decodes buf[0..len). Malformed input (an entry's declared length running +// past the end of buf) stops decoding at that point rather than reading out +// of bounds; entries decoded so far are still returned. +inline DecodedRelations DecodeRelations(const char* buf, uint16_t len) { + DecodedRelations out; + if (len == 0) { + return out; + } + // PSCH_RELATIONS_OOM_LEN: blob lost to DSA OOM, not zero relations. + if (len == PSCH_RELATIONS_OOM_LEN) { + out.truncated = true; + return out; + } + if (len < PSCH_RELATIONS_HEADER_LEN) { + return out; + } + + out.truncated = buf[0] != 0; + auto count = static_cast(buf[1]); + size_t pos = 2; + const size_t end = len; + + // Reads a [len:u8][bytes] field at pos, advancing it past the field. + // Returns false (leaving *field untouched) if len or bytes run past end. + auto read_field = [&](std::string_view* field) { + if (pos + 1 > end) { + return false; + } + auto field_len = static_cast(buf[pos]); + if (pos + 1 + field_len > end) { + return false; + } + *field = std::string_view(buf + pos + 1, field_len); + pos += 1 + field_len; + return true; + }; + + for (uint8_t i = 0; i < count; i++) { + std::string_view ns; + std::string_view name; + if (!read_field(&ns) || !read_field(&name) || pos + 1 > end) { + break; + } + std::string_view kind(buf + pos, 1); + pos += 1; + + out.namespaces.push_back(ns); + out.names.push_back(name); + out.kinds.push_back(kind); + } + + return out; +} + +#endif // PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ diff --git a/src/export/stats_exporter.cc b/src/export/stats_exporter.cc index 4710d9a..7352e7c 100644 --- a/src/export/stats_exporter.cc +++ b/src/export/stats_exporter.cc @@ -19,6 +19,7 @@ extern "C" { #include "export/exporter_interface.h" #include "export/otel_arrow_exporter.h" #include "export/otel_exporter.h" +#include "export/relations_decode.h" #include "export/sqlcommenter_parse.h" #include "export/stats_exporter.h" #include "queue/event.h" @@ -240,6 +241,16 @@ void ExportEventStatsInternal(PschEvent* event, int max_events, StatsExporter* e // Only declared when label tracking is on, so inserts keep the old column // set until the destination schema has `labels`. auto col_labels = psch_track_labels ? exporter->StatHCString("labels") : nullptr; + // Only declared when relations tracking is on, so inserts keep the old + // column set until the destination schema has these columns. + auto col_relation_namespaces = + psch_relations_enabled ? exporter->StatHCStringArray("relation_namespaces") : nullptr; + auto col_relation_names = + psch_relations_enabled ? exporter->StatHCStringArray("relation_names") : nullptr; + auto col_relation_kinds = + psch_relations_enabled ? exporter->StatHCStringArray("relation_kinds") : nullptr; + auto col_relations_truncated = + psch_relations_enabled ? exporter->StatLCUInt8("relations_truncated") : nullptr; auto col_shared_blks_hit = exporter->StatHCInt64("shared_blks_hit"); auto col_shared_blks_read = exporter->StatHCInt64("shared_blks_read"); @@ -305,6 +316,16 @@ void ExportEventStatsInternal(PschEvent* event, int max_events, StatsExporter* e col_labels->Append(QueryLabelsJson(query_text, PSCH_MAX_QUERY_LEN)); } + if (col_relation_namespaces != nullptr) { + auto rlen = ClampFieldLen(ev.relations_len, static_cast(PSCH_MAX_RELATIONS_LEN), + "relations_len"); + auto decoded = DecodeRelations(ev.relations, rlen); + col_relation_namespaces->Append(decoded.namespaces); + col_relation_names->Append(decoded.names); + col_relation_kinds->Append(decoded.kinds); + col_relations_truncated->Append(decoded.truncated ? 1 : 0); + } + col_shared_blks_hit->Append(ev.shared_blks_hit); col_shared_blks_read->Append(ev.shared_blks_read); col_shared_blks_dirtied->Append(ev.shared_blks_dirtied); diff --git a/src/hooks/hooks.c b/src/hooks/hooks.c index e3c39ad..278b4d7 100644 --- a/src/hooks/hooks.c +++ b/src/hooks/hooks.c @@ -6,12 +6,14 @@ #include "access/parallel.h" #include "access/xact.h" +#include "catalog/pg_class.h" #include "commands/dbcommands.h" #include "common/ip.h" #include "common/pg_prng.h" #include "executor/executor.h" #include "executor/instrument.h" #include "jit/jit.h" +#include "lib/stringinfo.h" #include "libpq/libpq-be.h" #include "miscadmin.h" #include "nodes/queryjumble.h" @@ -27,6 +29,8 @@ #include "compat.h" #include "config/guc.h" #include "hooks/hooks.h" +#include "hooks/relations_budget.h" +#include "hooks/relations_nesting.h" #include "hooks/query_normalize.h" #include "hooks/query_normalize_state.h" #include "hooks/string_utils.h" @@ -39,6 +43,7 @@ static ExecutorStart_hook_type prev_executor_start = NULL; static ExecutorRun_hook_type prev_executor_run = NULL; static ExecutorFinish_hook_type prev_executor_finish = NULL; static ExecutorEnd_hook_type prev_executor_end = NULL; +static ExecutorCheckPerms_hook_type prev_executor_check_perms = NULL; static ProcessUtility_hook_type prev_process_utility = NULL; static emit_log_hook_type prev_emit_log_hook = NULL; @@ -63,6 +68,25 @@ static bool system_init = false; static int GetClientAddress(char* buf, int buf_size); static void ResolveNames(PschEvent* event); +// Relations collection spans two hooks: ExecutorCheckPerms_hook stages a +// query's touched tables into one of these buffers, ExecutorEnd_hook reads +// and frees it. Buffers are keyed by QueryDesc* rather than call depth, and +// freed on transaction/subtransaction abort as well as normal completion -- +// see PschRelationsNestingXactCallback and PopRelationsNestingLevel for why. +#define PSCH_RELATIONS_MAX_NESTING_DEPTH 16 +// Zero-initialized by static storage duration, so a fresh slot's .data is +// NULL until PschExecutorCheckPerms calls initStringInfo on it. +static StringInfoData relations_bufs[PSCH_RELATIONS_MAX_NESTING_DEPTH]; +static PschRelationsNestingEntry relations_nesting_table[PSCH_RELATIONS_MAX_NESTING_DEPTH]; +static bool relations_nesting_callbacks_registered = false; + +// ExecutorCheckPerms_hook only receives the range table, not the QueryDesc; +// PschExecutorStart supplies this (see CallExecutorStartChain). NULL outside +// CallExecutorStartChain's call to standard_ExecutorStart -- that's what +// distinguishes a real ExecutorCheckPerms_hook call from COPY/RI_Initial_Check +// calling it directly, with no QueryDesc to key a reservation on. +static QueryDesc* relations_active_query_desc = NULL; + // Cache for session-stable values to avoid repeated catalog lookups on every query. // Following pg_stat_monitor's pattern of caching client IP (pg_stat_monitor.c:73-96). // Database name and client address never change within a session. Username is @@ -304,6 +328,60 @@ static void CopyQueryText(PschEvent* event, uint64 query_id) { sizeof(event->query), &event->query_len); } +// Free reservations an aborting (sub)transaction never released itself; a +// WITH HOLD cursor surviving a commit is reparented to level 0, same reason +// as PschRelationsNestingReparentCommitted's subxact-level callers. +static void PschRelationsNestingXactCallback(XactEvent event, void* arg pg_attribute_unused()) { + if (event == XACT_EVENT_ABORT || event == XACT_EVENT_PARALLEL_ABORT) { + PschRelationsNestingAbortTo(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, 1); + } else if (event == XACT_EVENT_COMMIT || event == XACT_EVENT_PARALLEL_COMMIT) { + PschRelationsNestingReparentCommitted(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, 1); + } +} + +static void PschRelationsNestingSubXactCallback(SubXactEvent event, + SubTransactionId my_subid pg_attribute_unused(), + SubTransactionId parent_subid pg_attribute_unused(), + void* arg pg_attribute_unused()) { + if (event == SUBXACT_EVENT_ABORT_SUB) { + PschRelationsNestingAbortTo(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, + GetCurrentTransactionNestLevel()); + } else if (event == SUBXACT_EVENT_COMMIT_SUB) { + PschRelationsNestingReparentCommitted(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, + GetCurrentTransactionNestLevel()); + } +} + +static void EnsureRelationsNestingCallbacksRegistered(void) { + if (relations_nesting_callbacks_registered) { + return; + } + RegisterXactCallback(PschRelationsNestingXactCallback, NULL); + RegisterSubXactCallback(PschRelationsNestingSubXactCallback, NULL); + relations_nesting_callbacks_registered = true; +} + +// Must run unconditionally on every ExecutorEnd call. +static int PopRelationsNestingLevel(QueryDesc* query_desc) { + return PschRelationsNestingPop(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, + query_desc); +} + +// slot < 0 means the nesting table was genuinely full (relations_enabled=off +// still reserves a slot, see PschExecutorCheckPerms). +static void CopyRelations(PschEvent* event, int slot) { + if (slot < 0) { + event->relations[0] = 1; // truncated + event->relations[1] = 0; // count + event->relations_len = PSCH_RELATIONS_HEADER_LEN; + return; + } + + Assert(relations_bufs[slot].len <= (int)sizeof(event->relations)); + memcpy(event->relations, relations_bufs[slot].data, relations_bufs[slot].len); + event->relations_len = (uint16)relations_bufs[slot].len; +} + // Resolve database and user names, using the session cache when available. // Falls back to catalog lookups if cache hasn't been initialized yet (e.g., // emit_log_hook fires before the first executor hook). @@ -371,7 +449,7 @@ static uint64 QueryDurationUs(QueryDesc* query_desc) { } static void BuildEventFromQueryDesc(QueryDesc* query_desc, PschEvent* event, int64 cpu_user_us, - int64 cpu_sys_us) { + int64 cpu_sys_us, int relations_slot) { InitBaseEvent(event, query_start_ts, current_query_is_top_level, ConvertCmdType(query_desc->operation)); event->queryid = query_desc->plannedstmt->queryId; @@ -392,6 +470,7 @@ static void BuildEventFromQueryDesc(QueryDesc* query_desc, PschEvent* event, int CopyParallelWorkerInfo(event, query_desc); CopyClientContext(event); CopyQueryText(event, query_desc->plannedstmt->queryId); + CopyRelations(event, relations_slot); } // post_parse_analyze_hook — decide query text at parse time. @@ -437,13 +516,28 @@ static void PschPostParseAnalyze(ParseState* pstate, Query* query, PschJumbleSta } } -static void PschExecutorStart(QueryDesc* query_desc, int eflags) { - if (IsParallelWorker()) { +// PG_FINALLY restores both fields even if the chain errors. +static void CallExecutorStartChain(QueryDesc* query_desc, int eflags) { + QueryDesc* prev_active_query_desc = relations_active_query_desc; + relations_active_query_desc = query_desc; + PG_TRY(); + { if (prev_executor_start != NULL) { prev_executor_start(query_desc, eflags); } else { standard_ExecutorStart(query_desc, eflags); } + } + PG_FINALLY(); + { + relations_active_query_desc = prev_active_query_desc; + } + PG_END_TRY(); +} + +static void PschExecutorStart(QueryDesc* query_desc, int eflags) { + if (IsParallelWorker()) { + CallExecutorStartChain(query_desc, eflags); return; } @@ -466,11 +560,7 @@ static void PschExecutorStart(QueryDesc* query_desc, int eflags) { } #endif - if (prev_executor_start != NULL) { - prev_executor_start(query_desc, eflags); - } else { - standard_ExecutorStart(query_desc, eflags); - } + CallExecutorStartChain(query_desc, eflags); #if PG_VERSION_NUM < 190000 if (track && query_desc->totaltime == NULL) { @@ -504,6 +594,12 @@ static void PschExecutorRun(QueryDesc* query_desc, ScanDirection direction, uint return; } + // Marks this query's reservation (if any) as used at the current level -- + // unconditional like Pop, so toggling psch_enabled mid-session can't orphan it. + int prev_active_level = PschRelationsNestingTouch(relations_nesting_table, + PSCH_RELATIONS_MAX_NESTING_DEPTH, query_desc, + GetCurrentTransactionNestLevel()); + nesting_level++; PG_TRY(); { @@ -520,6 +616,11 @@ static void PschExecutorRun(QueryDesc* query_desc, ScanDirection direction, uint standard_ExecutorRun(query_desc, direction, count, execute_once); } #endif + // Only on success -- an error skips this, leaving active_level raised. + if (prev_active_level >= 0) { + PschRelationsNestingSetActiveLevel(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, + query_desc, prev_active_level); + } } PG_FINALLY(); { nesting_level--; } @@ -551,6 +652,9 @@ static void PschExecutorFinish(QueryDesc* query_desc) { } static void PschExecutorEnd(QueryDesc* query_desc) { + // Unconditional: must run even on paths below that skip the export itself. + int relations_slot = PopRelationsNestingLevel(query_desc); + if (!psch_enabled || IsParallelWorker() || query_desc->plannedstmt->queryId == UINT64CONST(0)) { if (prev_executor_end != NULL) { prev_executor_end(query_desc); @@ -588,7 +692,7 @@ static void PschExecutorEnd(QueryDesc* query_desc) { } PschEvent event; - BuildEventFromQueryDesc(query_desc, &event, cpu_user_us, cpu_sys_us); + BuildEventFromQueryDesc(query_desc, &event, cpu_user_us, cpu_sys_us, relations_slot); PschEnqueueEvent(&event); if (prev_executor_end != NULL) { @@ -838,6 +942,141 @@ bool PschSuppressErrorCapture(bool suppress) { return prev; } +// True if relid is already in seen[0..nseen). +static bool AlreadySeen(const Oid* seen, int nseen, Oid relid) { + for (int i = 0; i < nseen; i++) { + if (seen[i] == relid) { + return true; + } + } + return false; +} + +// Builds the relations blob: [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count. +static void CollectRelations(StringInfo buf, List* rt) { + // Sized to PSCH_MAX_RELATIONS_LEN, not an arbitrary smaller constant: every + // entry consumes at least 1 byte of the relations_buf budget, so nseen can + // never exceed the byte budget (<= PSCH_MAX_RELATIONS_LEN - 1) before the + // WouldExceedBudget break below fires. A smaller cap here would let nseen + // saturate while relations_buf still has room, silently disabling dedup + // for later relations without disabling appends -- so a table referenced + // twice past that point would show up twice. + Oid seen[PSCH_MAX_RELATIONS_LEN]; + int nseen = 0; + uint8 count = 0; + bool truncated = false; + ListCell* lc; + + // Defense in depth -- the GUC's own min bound (guc.c) already enforces this. + Assert(psch_relations_max_len >= PSCH_RELATIONS_HEADER_LEN); + + // Reserve the header now; patched with real values once known. + appendStringInfoChar(buf, 0); + appendStringInfoChar(buf, 0); + + foreach (lc, rt) { + RangeTblEntry* rte = (RangeTblEntry*)lfirst(lc); + + // Plain tables, plus view-expansion artifacts: the rewriter turns a + // referenced view into an RTE_SUBQUERY wrapping its definition, but + // stamps relkind='v' on that entry so it's still identifiable as a view. + // RTE_RELATION alone already covers every other relkind (matview, + // partitioned, foreign table) uniformly -- unaffected by that work. + // Partition scans surface the scanned child partitions here, not the + // declared parent -- that's the planner's own range table, not this. + if (rte->rtekind != RTE_RELATION && rte->relkind != RELKIND_VIEW) { + continue; + } + + if (AlreadySeen(seen, nseen, rte->relid)) { + continue; + } + if (nseen < (int)(lengthof(seen))) { + seen[nseen++] = rte->relid; + } + Assert(nseen <= (int)(lengthof(seen))); + + const char* namespace_name = get_namespace_name(get_rel_namespace(rte->relid)); + const char* relation_name = get_rel_name(rte->relid); + if (namespace_name == NULL || relation_name == NULL) { + continue; // Relation dropped concurrently; skip rather than crash. + } + + // NAMEDATALEN-1 (63) max per Postgres identifier rules, so these always + // fit in a uint8 length prefix. + size_t ns_len = strlen(namespace_name); + size_t name_len = strlen(relation_name); + Assert(ns_len < NAMEDATALEN); + Assert(name_len < NAMEDATALEN); + + int entry_len = (int)(1 + ns_len + 1 + name_len + 1); + if (PschRelationsWouldExceedBudget(buf->len, entry_len, psch_relations_max_len)) { + truncated = true; + break; // Budget exhausted -- truncate, same as query text does. + } + + appendStringInfoChar(buf, (char)ns_len); + appendBinaryStringInfo(buf, namespace_name, (int)ns_len); + appendStringInfoChar(buf, (char)name_len); + appendBinaryStringInfo(buf, relation_name, (int)name_len); + appendStringInfoChar(buf, (char)rte->relkind); + count++; + } + + buf->data[0] = truncated ? 1 : 0; + buf->data[1] = (char)count; +} + +// Releases the slot on query-context teardown -- catches portal failures +// that happen before ExecutorRun, which level-based sweeps can miss. +typedef struct PschRelationsNestingReleaseCallback { + MemoryContextCallback cb; + const void* key; +} PschRelationsNestingReleaseCallback; + +static void PschRelationsNestingReleaseOnContextTeardown(void* arg) { + const PschRelationsNestingReleaseCallback* release_cb = (const PschRelationsNestingReleaseCallback*)arg; + PschRelationsNestingPop(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, release_cb->key); +} + +// ExecutorCheckPerms_hook -- collects relations, then defers the actual +// permission decision to the previous hook (or the standard "allow" default). +static bool PschExecutorCheckPerms(List* rt, List* rp, bool ereport_on_violation) { + // Reservation is gated on relations_active_query_desc only, not + // relations_enabled, which instead decides whether to actually collect. + if (psch_enabled && !IsParallelWorker() && relations_active_query_desc != NULL) { + EnsureRelationsNestingCallbacksRegistered(); + int slot = PschRelationsNestingPush(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, + relations_active_query_desc, + GetCurrentTransactionNestLevel()); + if (slot >= 0) { + PschRelationsNestingReleaseCallback* release_cb = + MemoryContextAlloc(CurrentMemoryContext, sizeof(*release_cb)); + release_cb->cb.func = PschRelationsNestingReleaseOnContextTeardown; + release_cb->cb.arg = release_cb; + release_cb->key = relations_active_query_desc; + MemoryContextRegisterResetCallback(CurrentMemoryContext, &release_cb->cb); + + if (relations_bufs[slot].data == NULL) { + MemoryContext oldcxt = MemoryContextSwitchTo(TopMemoryContext); + initStringInfo(&relations_bufs[slot]); + MemoryContextSwitchTo(oldcxt); + } else { + resetStringInfo(&relations_bufs[slot]); + } + + if (psch_relations_enabled) { + CollectRelations(&relations_bufs[slot], rt); + } + } + } + + if (prev_executor_check_perms != NULL) { + return prev_executor_check_perms(rt, rp, ereport_on_violation); + } + return true; +} + void PschInstallHooks(void) { EnableQueryId(); @@ -856,6 +1095,9 @@ void PschInstallHooks(void) { prev_executor_end = ExecutorEnd_hook; ExecutorEnd_hook = PschExecutorEnd; + prev_executor_check_perms = ExecutorCheckPerms_hook; + ExecutorCheckPerms_hook = PschExecutorCheckPerms; + prev_process_utility = ProcessUtility_hook; ProcessUtility_hook = PschProcessUtility; diff --git a/src/queue/event.h b/src/queue/event.h index 4f680be..af7f727 100644 --- a/src/queue/event.h +++ b/src/queue/event.h @@ -16,10 +16,8 @@ // 4. FAST COPY: memcpy() of 2KB is ~20ns on modern CPUs with cache. Variable-length // would require copying arbitrary amounts of data under lock. // -// ALTERNATIVE CONSIDERED: pg_stat_monitor uses DSA (Dynamic Shared Area) for -// variable-length query text. This allows unlimited query length but adds complexity -// with DSA allocation, OOM handling, and pointer management. For telemetry/metrics, -// 2KB query text is sufficient - we don't need full query text for aggregation. +// Each cap below also bounds how much of PschRingEntry's shared DSA pool one +// query can consume, so no single query can starve others. #ifndef PG_STAT_CH_SRC_QUEUE_EVENT_H_ #define PG_STAT_CH_SRC_QUEUE_EVENT_H_ @@ -45,6 +43,10 @@ extern "C" { // Maximum error message length (truncated if longer) #define PSCH_MAX_ERR_MSG_LEN 2048 +// Maximum length of the relations blob (layout in hooks.c, CollectRelations), +// truncated on overflow like query text -- truncation is self-flagged. +#define PSCH_MAX_RELATIONS_LEN 1024 + // Maximum application name length #define PSCH_MAX_APP_NAME_LEN (NAMEDATALEN - 1) @@ -163,11 +165,19 @@ typedef struct PschEvent { char client_addr[46]; // Client IP address (INET6_ADDRSTRLEN=46) uint8 client_addr_len; // Actual length uint16 query_len; // Actual length of query text + uint16 relations_len; // Actual length of relations blob (0 for utility statements) + // Keeps the fixed prefix's tail 8-byte aligned so it stays byte-identical + // to PschRingEntry's (dsa_pointer needs 8-byte alignment; char[] doesn't). + uint8 _padding4[6]; // Variable-length text fields — kept last so the fixed-size prefix can be // block-copied between PschEvent and PschRingEntry with a single memcpy. + // All copied as raw bytes with no encoding conversion -- assumes UTF-8. + // A real fix needs pg_do_encoding_conversion() per field plus a test + // matrix across non-UTF-8/ICU locales; left as a documented gap for now. char err_message[PSCH_MAX_ERR_MSG_LEN]; // Error message text (truncated if necessary) char query[PSCH_MAX_QUERY_LEN]; // Query text, unterminated, see query_len + char relations[PSCH_MAX_RELATIONS_LEN]; // See CollectRelations in hooks.c for layout. } PschEvent; // Ensure the struct has expected size characteristics diff --git a/src/queue/ring_entry.h b/src/queue/ring_entry.h index faf1c4c..72590fd 100644 --- a/src/queue/ring_entry.h +++ b/src/queue/ring_entry.h @@ -1,11 +1,11 @@ // Compact ring buffer entry for shared memory storage. // // PschRingEntry mirrors PschEvent's numeric fields identically but replaces -// the two large inline char arrays (query[2048] and err_message[2048]) with -// dsa_pointer values that reference variable-length strings allocated in a -// DSA (Dynamic Shared Memory Area). +// the large inline char arrays (query[2048], err_message[2048], and +// relations[1024]) with dsa_pointer values that reference variable-length +// strings allocated in a DSA (Dynamic Shared Memory Area). // -// This shrinks each ring buffer slot from ~4.5KB to ~500 bytes while keeping +// This shrinks each ring buffer slot from ~5KB to ~510 bytes while keeping // the public API type (PschEvent) unchanged. The conversion happens // transparently inside PschEnqueueEvent / PschDequeueEvent. #ifndef PG_STAT_CH_SRC_QUEUE_RING_ENTRY_H_ @@ -98,10 +98,13 @@ typedef struct PschRingEntry { char client_addr[46]; uint8 client_addr_len; uint16 query_len; + uint16 relations_len; + uint8 _padding4[6]; // Matches PschEvent's padding; see event.h for why. - // --- DSA pointers replace the two trailing char arrays in PschEvent --- + // --- DSA pointers replace the three trailing char arrays in PschEvent --- dsa_pointer err_message_dsa; // was: char err_message[PSCH_MAX_ERR_MSG_LEN] dsa_pointer query_dsa; // was: char query[PSCH_MAX_QUERY_LEN] + dsa_pointer relations_dsa; // was: char relations[PSCH_MAX_RELATIONS_LEN] } PschRingEntry; // Verify that fixed prefix of PschRingEntry has same binary layout as @@ -127,8 +130,10 @@ StaticAssertDecl(offsetof(PschEvent, application_name) == offsetof(PschRingEntry "application_name offset mismatch"); StaticAssertDecl(offsetof(PschEvent, query_len) == offsetof(PschRingEntry, query_len), "query_len offset mismatch"); -// Fields after query_len diverge: PschEvent has char arrays, PschRingEntry -// has dsa_pointers. Intentional, fixed prefix up to query_len is +StaticAssertDecl(offsetof(PschEvent, relations_len) == offsetof(PschRingEntry, relations_len), + "relations_len offset mismatch"); +// Fields after relations_len diverge: PschEvent has char arrays, PschRingEntry +// has dsa_pointers. Intentional, fixed prefix up to relations_len is // block-copied with single memcpy. StaticAssertDecl(offsetof(PschEvent, err_message) == offsetof(PschRingEntry, err_message_dsa), "err_message/err_message_dsa start offset mismatch"); diff --git a/src/queue/shmem.c b/src/queue/shmem.c index 92b9a86..95ca5b8 100644 --- a/src/queue/shmem.c +++ b/src/queue/shmem.c @@ -40,6 +40,7 @@ #include "config/guc.h" #include "hooks/hooks.h" +#include "hooks/relations_budget.h" #include "queue/local_batch.h" #include "queue/psch_dsa.h" #include "queue/query_intern.h" @@ -112,15 +113,23 @@ static bool TryEnqueueLocked(const PschEvent* event, uint32 capacity) { // client_addr, lengths — everything before the variable-length data). memcpy(slot, event, kFixedPrefixSize); - // 2. Allocate DSA for err_message and query text. Store pointers in slot - // only right before head advances, so slots outside [tail, head) never - // hold references + // 2. Allocate DSA for err_message, relations, and query text. Store + // pointers in slot only right before head advances, so slots outside + // [tail, head) never hold references dsa_pointer err_message_dsa = PschDsaAllocString(event->err_message, event->err_message_len, PSCH_MAX_ERR_MSG_LEN); if (event->err_message_len > 0 && !DsaPointerIsValid(err_message_dsa)) { slot->err_message_len = 0; // Lost string on OOM — numeric data preserved } + // Relations: plain per-event DSA allocation, same as err_message (not + // interned, unlike the query text below). + dsa_pointer relations_dsa = + PschDsaAllocString(event->relations, event->relations_len, PSCH_MAX_RELATIONS_LEN); + if (event->relations_len > 0 && !DsaPointerIsValid(relations_dsa)) { + slot->relations_len = PSCH_RELATIONS_OOM_LEN; + } + // Query text goes through the shared interner so repeated identical // normalized queries share a single DSA-allocated body. See query_intern.h // for the design rationale. On miss + DSA OOM, miss + hash-full, or hash @@ -139,6 +148,7 @@ static bool TryEnqueueLocked(const PschEvent* event, uint32 capacity) { } slot->err_message_dsa = err_message_dsa; + slot->relations_dsa = relations_dsa; slot->query_dsa = query_dsa; // CRITICAL: Memory barrier ensures the event data is written to shared memory @@ -174,14 +184,14 @@ static Size PschSharedBlockSize(void) { Size PschShmemSize(void) { // Total shmem requested from the postmaster: the shared block plus the - // interner HTAB (which ShmemInitHash carves out of the same pool). + // query-text interner HTAB (carved from the same pool by ShmemInitHash). return add_size(PschSharedBlockSize(), PschQueryInternShmemSize()); } static void RequestSharedResources(void) { RequestAddinShmemSpace(PschShmemSize()); - // 1 main queue lock + N partition locks for the query-text interner, all - // in the single "pg_stat_ch" named tranche so we read them as a contiguous + // 1 main queue lock + N query-text interner partition locks, all in the + // single "pg_stat_ch" named tranche so we read them as a contiguous // LWLockPadded[] from GetNamedLWLockTranche(). RequestNamedLWLockTranche("pg_stat_ch", 1 + PschQueryInternLockCount()); } @@ -257,7 +267,8 @@ static void PschShmemStartupHook(void) { InitializeSharedState(); } - // Initialize (or attach to) the query intern HTAB. All backends get same handle. + // Initialize (or attach to) the query intern HTAB. All backends get the + // same handle. LWLockPadded* lwlocks = GetNamedLWLockTranche("pg_stat_ch"); PschQueryInternShmemInit(&lwlocks[1]); @@ -443,6 +454,13 @@ bool PschDequeueEvent(PschEvent* event) { PschDsaResolveString(err_message_dsa, event->err_message_len, event->err_message, PSCH_MAX_ERR_MSG_LEN, &event->err_message_len); + dsa_pointer relations_dsa = slot->relations_dsa; + slot->relations_dsa = InvalidDsaPointer; + if (event->relations_len != PSCH_RELATIONS_OOM_LEN) { + PschDsaResolveString(relations_dsa, event->relations_len, event->relations, + PSCH_MAX_RELATIONS_LEN, &event->relations_len); + } + dsa_pointer query_dsa = slot->query_dsa; slot->query_dsa = InvalidDsaPointer; PschQueryInternResolveAndRelease(query_dsa, event->query_len, event->query, PSCH_MAX_QUERY_LEN, diff --git a/t/036_unified_arrow_e2e.pl b/t/036_unified_arrow_e2e.pl index c0e9cf8..a26e0c7 100644 --- a/t/036_unified_arrow_e2e.pl +++ b/t/036_unified_arrow_e2e.pl @@ -69,6 +69,7 @@ pg_stat_ch.otel_endpoint = 'localhost:14317' pg_stat_ch.otel_arrow_passthrough = on pg_stat_ch.use_unified_arrow_exporter = on +pg_stat_ch.relations_enabled = on pg_stat_ch.debug_arrow_dump_dir = '$dump_dir' pg_stat_ch.hostname = 'unified-arrow-e2e-host' pg_stat_ch.extra_attributes = 'instance_ubid:test-instance;server_role:primary;region:test-region;cell:test-cell;read_replica_type:none' @@ -121,14 +122,11 @@ 'http://localhost:18123/?query=' . 'INSERT%20INTO%20pg_stat_ch.events_raw%20FORMAT%20ArrowStream'; -my $total_posted = 0; -for my $f (@ipc_files) { +# --fail-with-body makes HTTP 4xx/5xx from CH surface as a non-zero curl exit +# code (default curl returns 0 on any completed transfer regardless of status). +sub post_ipc_file { + my ($f) = @_; my $err_file = "$f.err"; - # --fail-with-body makes HTTP 4xx/5xx responses from CH surface as a - # non-zero curl exit code (default curl returns 0 on any completed - # HTTP transfer regardless of status). The subsequent SELECT count() - # assertion would catch ingestion failure too, but failing here gives - # a sharper error message at the point the bug actually happened. my $rc = system("curl -sS --fail-with-body -X POST " . "-H 'Content-Type: application/octet-stream' " . "--data-binary \@$f '$insert_url' 2>$err_file >/dev/null"); @@ -137,6 +135,12 @@ } : ''; is($rc, 0, basename($f) . " accepted by CH ArrowStream parser" . ($stderr ? " (stderr: $stderr)" : "")); + return $rc == 0; +} + +my $total_posted = 0; +for my $f (@ipc_files) { + post_ipc_file($f); ++$total_posted; } cmp_ok($total_posted, '>=', 1, "Posted at least one IPC file to CH"); @@ -186,5 +190,47 @@ is($envelope, "test-instance\tprimary\ttest-region\ttest-cell\tnone", "envelope columns populated from extra_attributes"); +# A second, separately-flushed batch: checks the Arrow list-builder resets +# across flushes. Polls ClickHouse for the marker rather than local file state, +# since the bgworker's dump is async and not pinned to one file per flush. +my %posted = map { $_ => 1 } @ipc_files; + +$node->safe_psql('postgres', 'CREATE TABLE arrow_rel_t1 (id int)'); +$node->safe_psql('postgres', 'CREATE TABLE arrow_rel_t2 (id int)'); +$node->safe_psql('postgres', + 'SELECT arrow_rel_t1.id FROM arrow_rel_t1 JOIN arrow_rel_t2 ' . + 'ON arrow_rel_t1.id = arrow_rel_t2.id'); +$node->safe_psql('postgres', 'SELECT 99 AS arrow_rel_zero_marker'); +$node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + +my $rel_two = ''; +my $rel_zero = ''; +$deadline = time() + 10; +while (time() < $deadline) { + for my $f (glob("$dump_dir/*.ipc")) { + next if $posted{$f}; + $posted{$f} = 1; + post_ipc_file($f); + } + $rel_two ||= psch_query_clickhouse( + "SELECT relation_names, relation_kinds, relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE '%FROM arrow_rel_t1 JOIN arrow_rel_t2%' ORDER BY ts DESC LIMIT 1 FORMAT TSV"); + chomp $rel_two; + $rel_zero ||= psch_query_clickhouse( + "SELECT relation_names, relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE '%arrow_rel_zero_marker%' ORDER BY ts DESC LIMIT 1 FORMAT TSV"); + chomp $rel_zero; + last if $rel_two ne '' && $rel_zero ne ''; + select(undef, undef, undef, 0.2); +} + +like($rel_two, qr/'arrow_rel_t1'/, "two-relation query exported relation_names (got: $rel_two)"); +like($rel_two, qr/'arrow_rel_t2'/, "two-relation query exported both tables (got: $rel_two)"); +like($rel_two, qr/'r','r'/, "relation_kinds has two ordinary tables (got: $rel_two)"); +like($rel_two, qr/false$/, "two-relation query not truncated (got: $rel_two)"); + +is($rel_zero, "[]\tfalse", + "zero-relation query in the next batch shows an empty array, not leftover data"); + $node->stop(); done_testing(); diff --git a/t/039_relations_export.pl b/t/039_relations_export.pl new file mode 100644 index 0000000..bf75e30 --- /dev/null +++ b/t/039_relations_export.pl @@ -0,0 +1,785 @@ +#!/usr/bin/env perl +# Relations export: encoding correctness (identifiers, relkinds, truncation) +# and hook robustness under nested invocations and schema changes. +# Prerequisites: ClickHouse container must be running (docker compose -f +# docker/docker-compose.test.yml up -d); postgres_fdw gates the foreign-table case. + +use strict; +use warnings; +use lib 't'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use psch; + +if (!psch_clickhouse_available()) { + plan skip_all => 'Docker not available, skipping ClickHouse tests'; +} + +my $ch_check = `curl -s 'http://localhost:18123/' --data 'SELECT 1' 2>/dev/null`; +if ($ch_check !~ /^1/) { + plan skip_all => 'ClickHouse container not running. Start with: docker compose -f docker/docker-compose.test.yml up -d'; +} + +psch_query_clickhouse("TRUNCATE TABLE IF EXISTS pg_stat_ch.events_raw"); + +my $node = psch_init_node_with_clickhouse('relations_export', + flush_interval_ms => 100, + batch_max => 100 +); + +# relations_enabled is off by default and PGC_POSTMASTER (restart to change); +# this file's subtests assume it's on except the one that specifically tests +# the off/unmigrated case. +$node->append_conf('postgresql.conf', 'pg_stat_ch.relations_enabled = on'); +$node->restart(); + +$node->safe_psql('postgres', 'CREATE TABLE foo1(a int)'); +$node->safe_psql('postgres', 'CREATE TABLE foo2(b int)'); +$node->safe_psql('postgres', 'CREATE SCHEMA sch1'); +$node->safe_psql('postgres', 'CREATE TABLE sch1.foo3(c int)'); +$node->safe_psql('postgres', 'CREATE VIEW v1 AS SELECT * FROM foo1'); + +subtest 'plain table' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM foo1'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM foo1%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'foo1'/, "single table appears as foo1 (got: $names)"); +}; + +subtest 'two plain tables' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM foo1, foo2'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM foo1, foo2%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'foo1'/, "includes foo1 (got: $names)"); + like($names, qr/'foo2'/, "includes foo2 (got: $names)"); +}; + +subtest 'schema-qualified table' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM sch1.foo3, foo1'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $namespaces = psch_wait_for_clickhouse_query( + "SELECT relation_namespaces FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM sch1.foo3%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + my $names = psch_query_clickhouse( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM sch1.foo3%' LIMIT 1" + ); + like($namespaces, qr/'sch1'/, "sch1 namespace present (got: $namespaces)"); + like($names, qr/'foo3'/, "foo3 present (got: $names)"); + like($names, qr/'foo1'/, "foo1 also present (got: $names)"); +}; + +subtest 'self-join deduplicates' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM foo1 AS a, foo1 AS b'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM foo1 AS a%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + my $count = () = $names =~ /'foo1'/g; + is($count, 1, "foo1 listed once, not twice (got: $names)"); +}; + +subtest 'view reports relkind v' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM v1'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM v1%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + my $kinds = psch_query_clickhouse( + "SELECT relation_kinds FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM v1%' LIMIT 1" + ); + like($names, qr/'v1'/, "v1 present (got: $names)"); + like($kinds, qr/'v'/, "v1 reports relkind 'v' (got: $kinds)"); + like($names, qr/'foo1'/, "underlying base table also present (got: $names)"); +}; + +subtest 'truncation over budget is flagged and does not error' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # SET/RESET are session-local; safe_psql opens a fresh connection per + # call, so SET and the query it affects must run in the same call. + $node->safe_psql('postgres', + 'SET pg_stat_ch.relations_max_len = 8; ' . + 'SELECT * FROM foo1, foo2, sch1.foo3; ' . + 'RESET pg_stat_ch.relations_max_len'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $stats = psch_get_stats($node); + cmp_ok($stats->{dsa_oom}, '==', 0, 'no DSA OOM from the tiny budget'); + + my $ch_count = psch_wait_for_clickhouse_query( + "SELECT count() FROM pg_stat_ch.events_raw", + sub { $_[0] >= 1 }, + 10 + ); + cmp_ok($ch_count, '>=', 1, 'event still exported despite truncation'); + + my $truncated = psch_wait_for_clickhouse_query( + "SELECT relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1, foo2, sch1.foo3%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + is($truncated, 'true', "truncation is explicitly flagged (got: $truncated)"); + + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + $node->safe_psql('postgres', 'SELECT * FROM foo1, foo2, sch1.foo3'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $not_truncated = psch_wait_for_clickhouse_query( + "SELECT relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1, foo2, sch1.foo3%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + is($not_truncated, 'false', "complete list is explicitly not flagged (got: $not_truncated)"); +}; + +subtest 'minimum budget (header-only) produces an empty, truncated list' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # 2 is the new GUC floor -- room for the header, none for any entry. + $node->safe_psql('postgres', + 'SET pg_stat_ch.relations_max_len = 2; ' . + 'SELECT * FROM foo1; ' . + 'RESET pg_stat_ch.relations_max_len'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $stats = psch_get_stats($node); + cmp_ok($stats->{dsa_oom}, '==', 0, 'no DSA OOM at the minimum budget'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + is($names, '[]', "relation list is genuinely empty, not just under-filled (got: $names)"); + + my $truncated = psch_query_clickhouse( + "SELECT relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1%' LIMIT 1" + ); + is($truncated, 'true', "truncation is flagged even though zero entries fit (got: $truncated)"); +}; + +subtest 'relations_enabled=off reports empty but not truncated' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'ALTER SYSTEM SET pg_stat_ch.relations_enabled = off'); + $node->restart(); + + $node->safe_psql('postgres', 'SELECT * FROM foo1, foo2'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1, foo2%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + is($names, '[]', "relation list is empty while disabled (got: $names)"); + + my $truncated = psch_query_clickhouse( + "SELECT relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1, foo2%' LIMIT 1" + ); + is($truncated, 'false', "opt-out is not reported as truncation (got: $truncated)"); + + $node->safe_psql('postgres', 'ALTER SYSTEM SET pg_stat_ch.relations_enabled = on'); + $node->restart(); +}; + +# Encoding correctness: special characters, every relkind, the truncation +# marker, and the exact byte-budget boundary. + +subtest 'identifiers with special characters stay unambiguous' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE "weird.name"(b int)'); + $node->safe_psql('postgres', 'CREATE TABLE "has,comma"(c int)'); + $node->safe_psql('postgres', 'CREATE TABLE "has""quote"(d int)'); + $node->safe_psql('postgres', + 'SELECT * FROM "weird.name", "has,comma", "has""quote"'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE '%weird.name%has,comma%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + + # A comma or dot inside the name must not be treated as an element + # separator or namespace/name boundary. + like($names, qr/'has,comma'/, "has,comma survives as one element (got: $names)"); + like($names, qr/'weird\.name'/, "weird.name survives as one element, dot intact (got: $names)"); + like($names, qr/'has"quote'/, 'has"quote survives with embedded quote intact (got: ' . $names . ')'); + + # Let ClickHouse count array elements itself -- counting raw commas in + # the text form is wrong, since 'has,comma' has a literal comma inside + # its own quoted element, not just between elements. + my $count = psch_query_clickhouse( + "SELECT length(relation_names) FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE '%weird.name%has,comma%' LIMIT 1" + ); + is($count, '3', "exactly 3 elements, no extra split from the embedded comma (got names: $names)"); +}; + +subtest 'all relkinds are reported distinctly' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # postgres_fdw is a contrib module; CI's TAP-enabled PostgreSQL build + # (./configure --enable-tap-tests && make && make install, no + # `make -C contrib install`) does not build it, so foreign-table + # coverage degrades gracefully instead of hard-failing there. + my $has_fdw = $node->safe_psql('postgres', + "SELECT 1 FROM pg_available_extensions WHERE name = 'postgres_fdw'"); + + $node->safe_psql('postgres', 'CREATE TABLE rk_table(a int)'); + $node->safe_psql('postgres', 'CREATE VIEW rk_view AS SELECT * FROM rk_table'); + $node->safe_psql('postgres', 'CREATE MATERIALIZED VIEW rk_matview AS SELECT * FROM rk_table'); + $node->safe_psql('postgres', + 'CREATE TABLE rk_part(id int, val text) PARTITION BY RANGE (id)'); + $node->safe_psql('postgres', + 'CREATE TABLE rk_part_child PARTITION OF rk_part FOR VALUES FROM (0) TO (100)'); + + my $touch_tables = 'rk_table, rk_view, rk_matview, rk_part'; + if ($has_fdw eq '1') { + $node->safe_psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgres_fdw'); + $node->safe_psql('postgres', + "CREATE SERVER rk_loopback FOREIGN DATA WRAPPER postgres_fdw " . + "OPTIONS (host '" . $node->host . "', port '" . $node->port . "', dbname 'postgres')"); + $node->safe_psql('postgres', + "CREATE USER MAPPING FOR CURRENT_USER SERVER rk_loopback"); + $node->safe_psql('postgres', + "CREATE FOREIGN TABLE rk_ftable(a int) SERVER rk_loopback OPTIONS (table_name 'rk_table')"); + $touch_tables .= ', rk_ftable'; + } + + $node->safe_psql('postgres', "SELECT * FROM $touch_tables"); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $kinds = psch_wait_for_clickhouse_query( + "SELECT relation_kinds FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE '%rk_table, rk_view%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + + like($kinds, qr/'r'/, "plain table reports relkind 'r' (got: $kinds)"); + like($kinds, qr/'v'/, "view reports relkind 'v' (got: $kinds)"); + like($kinds, qr/'m'/, "materialized view reports relkind 'm' (got: $kinds)"); + like($kinds, qr/'p'/, "partitioned table reports relkind 'p' (got: $kinds)"); + + SKIP: { + skip 'postgres_fdw not available in this PostgreSQL build', 1 + unless $has_fdw eq '1'; + like($kinds, qr/'f'/, "foreign table reports relkind 'f' (got: $kinds)"); + } +}; + +subtest 'maximum-length identifier round-trips intact' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # NAMEDATALEN-1 (63) is the real max a Postgres identifier can hold -- + # anything longer is silently truncated by Postgres itself at parse time. + my $max_name = 'a' x 63; + $node->safe_psql('postgres', "CREATE TABLE $max_name(x int)"); + $node->safe_psql('postgres', "SELECT * FROM $max_name"); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE length(relation_names) > 0 ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/\Q$max_name\E/, "63-byte name round-trips without truncation or corruption (got: $names)"); +}; + +subtest 'identifier made entirely of quote characters round-trips intact' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # Craig's example: a table name that is itself all double-quote + # characters. A length-prefixed field needs no escaping for this -- + # it's just 63 raw 0x22 bytes with an explicit length prefix. + my $quote_name = '"' x 63; # the real, unquoted name + my $sql_literal = '"' . ('""' x 63) . '"'; # how to spell it in SQL + $node->safe_psql('postgres', "CREATE TABLE $sql_literal(x int)"); + $node->safe_psql('postgres', "SELECT * FROM $sql_literal"); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE length(relation_names) > 0 ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/\Q$quote_name\E/, "all-quote-character name survives intact, 63 raw bytes (got: $names)"); +}; + +subtest 'exactly 7 max-length relations fit the 1023-byte budget, the 8th is dropped' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # Worst-case per-relation cost: 1(ns_len) + 63(ns) + 1(name_len) + + # 63(name) + 1(relkind) = 129 bytes. (1023 - 2-byte header) / 129 = 7.9, + # so 7 fit and an 8th must be dropped with the truncated flag set. + my $ns63 = 'n' x 63; + $node->safe_psql('postgres', "CREATE SCHEMA $ns63"); + my @tables; + for my $i (1 .. 8) { + my $tname = ('r' x 61) . sprintf('%02d', $i); + push @tables, "$ns63.$tname"; + $node->safe_psql('postgres', "CREATE TABLE $ns63.$tname(x int)"); + } + $node->safe_psql('postgres', 'SELECT * FROM ' . join(', ', @tables)); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $count = psch_wait_for_clickhouse_query( + "SELECT length(relation_names) FROM pg_stat_ch.events_raw WHERE length(relation_names) > 0 ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + is($count, '7', "exactly 7 of 8 max-length relations fit the 1023-byte budget (got: $count)"); + + my $truncated = psch_query_clickhouse( + "SELECT relations_truncated FROM pg_stat_ch.events_raw WHERE length(relation_names) > 0 ORDER BY ts DESC LIMIT 1" + ); + is($truncated, 'true', "the dropped 8th relation is explicitly flagged (got: $truncated)"); +}; + +# Hook robustness: nested executor invocations (PL/pgSQL, dynamic EXECUTE, +# nested PL/pgSQL) must not clobber the outer statement's relations. + +$node->safe_psql('postgres', 'CREATE TABLE outer_t(a int)'); +$node->safe_psql('postgres', 'CREATE TABLE inner_t(b int)'); +$node->safe_psql('postgres', q{ + CREATE FUNCTION touches_inner(x int) RETURNS bigint AS $BODY$ + SELECT count(*) FROM inner_t; + $BODY$ LANGUAGE sql STRICT; +}); +# Must have real rows: the per-row function call -- and the nested executor +# invocation it triggers -- never happens at all against an empty table. +$node->safe_psql('postgres', 'INSERT INTO outer_t VALUES (1), (2)'); +$node->safe_psql('postgres', 'INSERT INTO inner_t VALUES (10)'); + +subtest 'nested executor invocation does not clobber the outer statement relations' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT a, touches_inner(a) FROM outer_t'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT a, touches_inner(a) FROM outer_t%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'outer_t'/, "outer statement reports outer_t, not clobbered by the nested call (got: $names)"); + unlike($names, qr/'inner_t'/, "outer statement's own relations do not include inner_t (got: $names)"); +}; + +$node->safe_psql('postgres', q{ + CREATE FUNCTION plpgsql_touches_inner(x int) RETURNS bigint AS $BODY$ + DECLARE + cnt bigint; + BEGIN + SELECT count(*) INTO cnt FROM inner_t; + RETURN cnt; + END; + $BODY$ LANGUAGE plpgsql; +}); + +subtest 'PL/pgSQL function body does not clobber the caller relations' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT a, plpgsql_touches_inner(a) FROM outer_t'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT a, plpgsql_touches_inner(a) FROM outer_t%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'outer_t'/, "outer statement reports outer_t (got: $names)"); + unlike($names, qr/'inner_t'/, "not clobbered by the PL/pgSQL function's own query (got: $names)"); +}; + +$node->safe_psql('postgres', q{ + CREATE FUNCTION plpgsql_touches_inner_dynamic(x int) RETURNS bigint AS $BODY$ + DECLARE + cnt bigint; + BEGIN + EXECUTE 'SELECT count(*) FROM inner_t' INTO cnt; + RETURN cnt; + END; + $BODY$ LANGUAGE plpgsql; +}); + +subtest 'PL/pgSQL dynamic EXECUTE does not clobber the caller relations' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT a, plpgsql_touches_inner_dynamic(a) FROM outer_t'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT a, plpgsql_touches_inner_dynamic(a) FROM outer_t%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'outer_t'/, "outer statement reports outer_t (got: $names)"); + unlike($names, qr/'inner_t'/, "not clobbered by the dynamic EXECUTE's query (got: $names)"); +}; + +$node->safe_psql('postgres', 'CREATE TABLE inner_t2(c int)'); +$node->safe_psql('postgres', 'INSERT INTO inner_t2 VALUES (20)'); +$node->safe_psql('postgres', q{ + CREATE FUNCTION plpgsql_level2() RETURNS bigint AS $BODY$ + DECLARE + cnt bigint; + BEGIN + SELECT count(*) INTO cnt FROM inner_t2; + RETURN cnt; + END; + $BODY$ LANGUAGE plpgsql; +}); +$node->safe_psql('postgres', q{ + CREATE FUNCTION plpgsql_level1(x int) RETURNS bigint AS $BODY$ + DECLARE + cnt bigint; + BEGIN + SELECT count(*) INTO cnt FROM inner_t; + RETURN cnt + plpgsql_level2(); + END; + $BODY$ LANGUAGE plpgsql; +}); + +subtest 'nested PL/pgSQL calling PL/pgSQL does not clobber the caller relations' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT a, plpgsql_level1(a) FROM outer_t'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT a, plpgsql_level1(a) FROM outer_t%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'outer_t'/, "outer statement reports outer_t (got: $names)"); + unlike($names, qr/'inner_t'/, "not clobbered by the first level of PL/pgSQL nesting (got: $names)"); + unlike($names, qr/'inner_t2'/, "not clobbered by the second level of PL/pgSQL nesting (got: $names)"); +}; + +subtest 'extension invoking SPI does not clobber the caller relations' => sub { + # tablefunc is a contrib module; CI's TAP-enabled PostgreSQL build + # (./configure --enable-tap-tests && make && make install, no + # `make -C contrib install`) does not build it, same gap as postgres_fdw + # above. + my $has_tablefunc = $node->safe_psql('postgres', + "SELECT 1 FROM pg_available_extensions WHERE name = 'tablefunc'"); + + SKIP: { + skip 'tablefunc not available in this PostgreSQL build', 2 + unless $has_tablefunc eq '1'; + + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE EXTENSION IF NOT EXISTS tablefunc'); + $node->safe_psql('postgres', q{ + SELECT a FROM outer_t, + crosstab('SELECT 1, 1, b FROM inner_t') AS ct(row_name int, val int) + }); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT a FROM outer_t,%crosstab%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'outer_t'/, "outer statement reports outer_t (got: $names)"); + unlike($names, qr/'inner_t'/, "not clobbered by crosstab's internal SPI query (got: $names)"); + } +}; + +# Hook robustness: relation drop/recreate and rename must not leave stale +# state behind in CollectRelations's catalog-lookup path. + +subtest 'table dropped and recreated with the same name reports cleanly' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE drtab(a int)'); + $node->safe_psql('postgres', 'SELECT * FROM drtab'); + $node->safe_psql('postgres', 'DROP TABLE drtab'); + $node->safe_psql('postgres', 'CREATE TABLE drtab(b text)'); + $node->safe_psql('postgres', 'SELECT * FROM drtab'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $stats = psch_get_stats($node); + cmp_ok($stats->{dsa_oom}, '==', 0, 'no crash/OOM across drop-and-recreate'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM drtab%' ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'drtab'/, "post-recreate query still reports drtab cleanly (got: $names)"); +}; + +subtest 'renamed table is reported under its new name' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE old_name_tab(a int)'); + $node->safe_psql('postgres', 'ALTER TABLE old_name_tab RENAME TO new_name_tab'); + $node->safe_psql('postgres', 'SELECT * FROM new_name_tab'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM new_name_tab%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'new_name_tab'/, "query by the new name reports new_name_tab (got: $names)"); + unlike($names, qr/'old_name_tab'/, "no stale old name lingers anywhere (got: $names)"); +}; + +subtest 'relations nesting depth recovers after a run of unsampled queries' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE nest_recover_t(a int)'); + + # 20 deterministically-unsampled queries (sample_rate=0, high + # min_duration_us) used to permanently saturate the nesting-depth stack. + my $sql = 'SET pg_stat_ch.sample_rate = 0.0; ' . + 'SET pg_stat_ch.min_duration_us = 2000000000; ' . + ('SELECT * FROM nest_recover_t; ' x 20) . + 'RESET pg_stat_ch.sample_rate; ' . + 'RESET pg_stat_ch.min_duration_us; ' . + 'SELECT * FROM nest_recover_t;'; + $node->safe_psql('postgres', $sql); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM nest_recover_t%' ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'nest_recover_t'/, + "relations still captured after a 20-query unsampled streak (got: $names)"); +}; + +subtest 'nesting depth recovers after permission-check failures inside a PL/pgSQL exception' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE perm_fail_t(a int)'); + $node->safe_psql('postgres', 'CREATE ROLE perm_fail_role'); + + # Each iteration fails inside ExecutorStart itself (permission denied, + # before any execution), caught by the PL/pgSQL exception block -- which + # runs it as its own subtransaction. 20 of these used to leak a nesting + # level every time with no way to recover. + my $sql = + 'SET ROLE perm_fail_role; ' . + q{DO $$ BEGIN + FOR i IN 1..20 LOOP + BEGIN + EXECUTE 'SELECT * FROM perm_fail_t'; + EXCEPTION WHEN insufficient_privilege THEN NULL; + END; + END LOOP; + END $$;} . + ' RESET ROLE; ' . + 'SELECT * FROM perm_fail_t;'; + $node->safe_psql('postgres', $sql); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM perm_fail_t%' ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'perm_fail_t'/, + "relations still captured after 20 permission-check failures (got: $names)"); +}; + +subtest 'nesting depth recovers after a run of COPY statements' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE copy_leak_t(a int)'); + + # COPY's own permission check bypasses ExecutorStart/End entirely, so 20 + # of these used to leak a nesting level every time with no way to recover. + my $sql = ("COPY copy_leak_t TO '/dev/null'; " x 20) . + 'SELECT * FROM copy_leak_t;'; + $node->safe_psql('postgres', $sql); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM copy_leak_t%' ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'copy_leak_t'/, + "relations still captured after 20 COPY statements (got: $names)"); +}; + +subtest 'nesting depth recovers after many distinct failed-cursor fetches' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE cursor_leak_t(a int)'); + + # 20 distinct failed-cursor/savepoint sequences used to exhaust all 16 + # nesting slots; must run in one session (psql, on_error_stop => 0), + # since relations_nesting_table is per-backend state. + my $sql = "BEGIN;\n"; + for my $i (1 .. 20) { + $sql .= "DECLARE cursor_leak_cur_$i NO SCROLL CURSOR FOR SELECT * FROM cursor_leak_t;\n"; + $sql .= "SAVEPOINT cursor_leak_sp_$i;\n"; + $sql .= "FETCH BACKWARD FROM cursor_leak_cur_$i;\n"; + $sql .= "ROLLBACK TO SAVEPOINT cursor_leak_sp_$i;\n"; + $sql .= "RELEASE SAVEPOINT cursor_leak_sp_$i;\n"; + } + $sql .= "COMMIT;\n"; + $sql .= "SELECT * FROM cursor_leak_t;\n"; + $sql .= "SELECT pg_stat_ch_flush();\n"; + $node->psql('postgres', $sql, on_error_stop => 0); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM cursor_leak_t%' ORDER BY ts DESC LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'cursor_leak_t'/, + "relations still captured after 20 distinct failed-cursor fetches (got: $names)"); +}; + +subtest 'relation_* columns are omitted when disabled, so an unmigrated schema still accepts inserts' => sub { + # Simulate a destination that hasn't run the relation_* columns + # migration yet (the Down side of 20261104000003_*): the exporter must + # not mention these columns when relations_enabled is off, or the whole + # insert -- not just these fields -- would be rejected. + psch_query_clickhouse("DROP VIEW pg_stat_ch.events_recent_1h"); + psch_query_clickhouse( + "ALTER TABLE pg_stat_ch.events_raw " . + "DROP COLUMN relation_namespaces, " . + "DROP COLUMN relation_names, " . + "DROP COLUMN relation_kinds, " . + "DROP COLUMN relations_truncated" + ); + psch_query_clickhouse( + "CREATE MATERIALIZED VIEW pg_stat_ch.events_recent_1h " . + "ENGINE = MergeTree " . + "PARTITION BY toDate(ts) " . + "ORDER BY (instance_ubid, ts) " . + "TTL toDateTime(ts) + INTERVAL 1 HOUR DELETE " . + "POPULATE " . + "AS SELECT * FROM pg_stat_ch.events_raw WHERE ts > now() - INTERVAL 1 HOUR" + ); + + $node->safe_psql('postgres', 'ALTER SYSTEM SET pg_stat_ch.relations_enabled = off'); + $node->restart(); + + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM foo1'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $duration = psch_wait_for_clickhouse_query( + "SELECT duration_us FROM pg_stat_ch.events_raw " . + "WHERE query_text LIKE 'SELECT * FROM foo1%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + isnt($duration, '', + "row still inserted against a schema missing the relation_* columns (got duration_us: $duration)"); + + # Restore the schema and re-enable, leaving the shared CH container + # clean for whatever test file runs after this one. + psch_query_clickhouse("DROP VIEW pg_stat_ch.events_recent_1h"); + psch_query_clickhouse( + "ALTER TABLE pg_stat_ch.events_raw " . + "ADD COLUMN relation_namespaces Array(LowCardinality(String)), " . + "ADD COLUMN relation_names Array(LowCardinality(String)), " . + "ADD COLUMN relation_kinds Array(FixedString(1)), " . + "ADD COLUMN relations_truncated Bool DEFAULT false" + ); + psch_query_clickhouse( + "CREATE MATERIALIZED VIEW pg_stat_ch.events_recent_1h " . + "ENGINE = MergeTree " . + "PARTITION BY toDate(ts) " . + "ORDER BY (instance_ubid, ts) " . + "TTL toDateTime(ts) + INTERVAL 1 HOUR DELETE " . + "POPULATE " . + "AS SELECT * FROM pg_stat_ch.events_raw WHERE ts > now() - INTERVAL 1 HOUR" + ); + + $node->safe_psql('postgres', 'ALTER SYSTEM SET pg_stat_ch.relations_enabled = on'); + $node->restart(); +}; + +$node->stop(); +done_testing(); diff --git a/test/regression/expected/guc.out b/test/regression/expected/guc.out index 1a09997..c81ea70 100644 --- a/test/regression/expected/guc.out +++ b/test/regression/expected/guc.out @@ -29,12 +29,14 @@ SELECT name FROM pg_settings WHERE name LIKE 'pg_stat_ch.%' ORDER BY name COLLAT pg_stat_ch.otel_max_block_bytes pg_stat_ch.otel_metric_interval_ms pg_stat_ch.queue_capacity + pg_stat_ch.relations_enabled + pg_stat_ch.relations_max_len pg_stat_ch.sample_rate pg_stat_ch.string_area_size pg_stat_ch.track_labels pg_stat_ch.use_otel pg_stat_ch.use_unified_arrow_exporter -(31 rows) +(33 rows) SHOW pg_stat_ch.enabled; pg_stat_ch.enabled diff --git a/test/regression/expected/relations.out b/test/regression/expected/relations.out new file mode 100644 index 0000000..1b2b38d --- /dev/null +++ b/test/regression/expected/relations.out @@ -0,0 +1,117 @@ +-- Relations array collection. +-- Actual captured VALUES (which relations landed in the array) can only be +-- verified against ClickHouse -- see t/039_relations_export.pl. This +-- regression test covers what pg_regress *can* check synchronously: the GUC, +-- and that ExecutorCheckPerms_hook doesn't crash/error across the basic +-- shapes a per-query relations capture needs to handle (plain tables, +-- schema-qualified tables, self-joins, views). +CREATE EXTENSION pg_stat_ch; +SELECT pg_stat_ch_reset(); + pg_stat_ch_reset +------------------ + +(1 row) + +-- GUC: registered, default, and settable within its bounds. +SHOW pg_stat_ch.relations_max_len; + pg_stat_ch.relations_max_len +------------------------------ + 1023 +(1 row) + +SET pg_stat_ch.relations_max_len = 64; +SHOW pg_stat_ch.relations_max_len; + pg_stat_ch.relations_max_len +------------------------------ + 64 +(1 row) + +RESET pg_stat_ch.relations_max_len; +-- The minimum (2 bytes -- just enough for the header) must still work +-- without error; see PSCH_RELATIONS_HEADER_LEN. +SET pg_stat_ch.relations_max_len = 2; +SHOW pg_stat_ch.relations_max_len; + pg_stat_ch.relations_max_len +------------------------------ + 2 +(1 row) + +RESET pg_stat_ch.relations_max_len; +CREATE TABLE rel_foo1(a int); +CREATE TABLE rel_foo2(b int); +CREATE SCHEMA rel_sch1; +CREATE TABLE rel_sch1.foo3(c int); +CREATE VIEW rel_v1 AS SELECT * FROM rel_foo1; +-- Plain tables, schema-qualified table, self-join, view. Only checking +-- these execute without error; see the TAP test for the actual captured +-- relations values. +SELECT * FROM rel_foo1; + a +--- +(0 rows) + +SELECT * FROM rel_foo1, rel_foo2; + a | b +---+--- +(0 rows) + +SELECT * FROM rel_sch1.foo3, rel_foo1; + c | a +---+--- +(0 rows) + +SELECT * FROM rel_foo1 AS a, rel_foo1 AS b; + a | a +---+--- +(0 rows) + +SELECT * FROM rel_v1; + a +--- +(0 rows) + +-- A query touching more relations than a tiny budget allows must not error -- +-- truncation, not failure. +SET pg_stat_ch.relations_max_len = 8; +SELECT * FROM rel_foo1, rel_foo2, rel_sch1.foo3; + a | b | c +---+---+--- +(0 rows) + +RESET pg_stat_ch.relations_max_len; +-- A FETCH that fails before the executor runs (backward fetch on a NO +-- SCROLL cursor, declared before an aborting savepoint) must not crash or +-- break relations tracking for the rest of the session. +BEGIN; +DECLARE rel_cur NO SCROLL CURSOR FOR SELECT * FROM rel_foo1; +SAVEPOINT rel_sp1; +FETCH BACKWARD FROM rel_cur; +ERROR: cursor can only scan forward +HINT: Declare it with SCROLL option to enable backward scan. +ROLLBACK TO SAVEPOINT rel_sp1; +CLOSE rel_cur; +COMMIT; +-- Relations tracking still works for a normal query afterward. +SELECT * FROM rel_foo1, rel_foo2; + a | b +---+--- +(0 rows) + +-- relations_enabled is PGC_POSTMASTER (see t/039_relations_export.pl for the +-- disabled-collection behavior, which needs a restart to toggle). +SELECT enqueued_events > 0 AS has_enqueued_events FROM pg_stat_ch_stats(); + has_enqueued_events +--------------------- + t +(1 row) + +SELECT dsa_oom_count = 0 AS has_no_dsa_ooms FROM pg_stat_ch_stats(); + has_no_dsa_ooms +----------------- + t +(1 row) + +DROP VIEW rel_v1; +DROP TABLE rel_foo1, rel_foo2, rel_sch1.foo3; +DROP SCHEMA rel_sch1; +DROP EXTENSION pg_stat_ch; diff --git a/test/regression/pg_stat_ch.conf b/test/regression/pg_stat_ch.conf index 8b3d654..2480caa 100644 --- a/test/regression/pg_stat_ch.conf +++ b/test/regression/pg_stat_ch.conf @@ -1,2 +1,3 @@ shared_preload_libraries = 'pg_stat_ch' pg_stat_ch.enabled = on +pg_stat_ch.relations_enabled = on diff --git a/test/regression/sql/relations.sql b/test/regression/sql/relations.sql new file mode 100644 index 0000000..ac13296 --- /dev/null +++ b/test/regression/sql/relations.sql @@ -0,0 +1,67 @@ +-- Relations array collection. +-- Actual captured VALUES (which relations landed in the array) can only be +-- verified against ClickHouse -- see t/039_relations_export.pl. This +-- regression test covers what pg_regress *can* check synchronously: the GUC, +-- and that ExecutorCheckPerms_hook doesn't crash/error across the basic +-- shapes a per-query relations capture needs to handle (plain tables, +-- schema-qualified tables, self-joins, views). +CREATE EXTENSION pg_stat_ch; +SELECT pg_stat_ch_reset(); + +-- GUC: registered, default, and settable within its bounds. +SHOW pg_stat_ch.relations_max_len; +SET pg_stat_ch.relations_max_len = 64; +SHOW pg_stat_ch.relations_max_len; +RESET pg_stat_ch.relations_max_len; + +-- The minimum (2 bytes -- just enough for the header) must still work +-- without error; see PSCH_RELATIONS_HEADER_LEN. +SET pg_stat_ch.relations_max_len = 2; +SHOW pg_stat_ch.relations_max_len; +RESET pg_stat_ch.relations_max_len; + +CREATE TABLE rel_foo1(a int); +CREATE TABLE rel_foo2(b int); +CREATE SCHEMA rel_sch1; +CREATE TABLE rel_sch1.foo3(c int); +CREATE VIEW rel_v1 AS SELECT * FROM rel_foo1; + +-- Plain tables, schema-qualified table, self-join, view. Only checking +-- these execute without error; see the TAP test for the actual captured +-- relations values. +SELECT * FROM rel_foo1; +SELECT * FROM rel_foo1, rel_foo2; +SELECT * FROM rel_sch1.foo3, rel_foo1; +SELECT * FROM rel_foo1 AS a, rel_foo1 AS b; +SELECT * FROM rel_v1; + +-- A query touching more relations than a tiny budget allows must not error -- +-- truncation, not failure. +SET pg_stat_ch.relations_max_len = 8; +SELECT * FROM rel_foo1, rel_foo2, rel_sch1.foo3; +RESET pg_stat_ch.relations_max_len; + +-- A FETCH that fails before the executor runs (backward fetch on a NO +-- SCROLL cursor, declared before an aborting savepoint) must not crash or +-- break relations tracking for the rest of the session. +BEGIN; +DECLARE rel_cur NO SCROLL CURSOR FOR SELECT * FROM rel_foo1; +SAVEPOINT rel_sp1; +FETCH BACKWARD FROM rel_cur; +ROLLBACK TO SAVEPOINT rel_sp1; +CLOSE rel_cur; +COMMIT; + +-- Relations tracking still works for a normal query afterward. +SELECT * FROM rel_foo1, rel_foo2; + +-- relations_enabled is PGC_POSTMASTER (see t/039_relations_export.pl for the +-- disabled-collection behavior, which needs a restart to toggle). + +SELECT enqueued_events > 0 AS has_enqueued_events FROM pg_stat_ch_stats(); +SELECT dsa_oom_count = 0 AS has_no_dsa_ooms FROM pg_stat_ch_stats(); + +DROP VIEW rel_v1; +DROP TABLE rel_foo1, rel_foo2, rel_sch1.foo3; +DROP SCHEMA rel_sch1; +DROP EXTENSION pg_stat_ch; diff --git a/test/unit/otel_string_array_attr_test.cc b/test/unit/otel_string_array_attr_test.cc new file mode 100644 index 0000000..77b04a9 --- /dev/null +++ b/test/unit/otel_string_array_attr_test.cc @@ -0,0 +1,61 @@ +#include "export/otel_string_array_attr.h" + +#include + +using opentelemetry::proto::common::v1::KeyValue; + +TEST(SetStringArrayAttr, KeyIsSet) { + KeyValue kv; + psch::SetStringArrayAttr(&kv, "relation_names", {}); + EXPECT_EQ(kv.key(), "relation_names"); +} + +TEST(SetStringArrayAttr, EmptyInputYieldsEmptyArrayNotAbsentAttr) { + KeyValue kv; + psch::SetStringArrayAttr(&kv, "relation_names", {}); + ASSERT_TRUE(kv.value().has_array_value()); + EXPECT_EQ(kv.value().array_value().values_size(), 0); +} + +TEST(SetStringArrayAttr, SingleEntry) { + KeyValue kv; + psch::SetStringArrayAttr(&kv, "relation_names", {"public.foo1"}); + ASSERT_EQ(kv.value().array_value().values_size(), 1); + EXPECT_EQ(kv.value().array_value().values(0).string_value(), "public.foo1"); +} + +TEST(SetStringArrayAttr, MultipleEntriesPreserveOrder) { + KeyValue kv; + psch::SetStringArrayAttr(&kv, "relation_names", {"public.foo1", "public.foo2", "sch.foo3"}); + const auto& arr = kv.value().array_value(); + ASSERT_EQ(arr.values_size(), 3); + EXPECT_EQ(arr.values(0).string_value(), "public.foo1"); + EXPECT_EQ(arr.values(1).string_value(), "public.foo2"); + EXPECT_EQ(arr.values(2).string_value(), "sch.foo3"); +} + +TEST(SetStringArrayAttr, SingleCharElementSurvives) { + KeyValue kv; + psch::SetStringArrayAttr(&kv, "relation_kinds", {"r"}); + EXPECT_EQ(kv.value().array_value().values(0).string_value(), "r"); +} + +TEST(EstimateStringArrayAttrBytes, GrowsWithElementCountAndLength) { + size_t empty = psch::EstimateStringArrayAttrBytes("relation_names", {}); + size_t one_short = psch::EstimateStringArrayAttrBytes("relation_names", {"a.b"}); + size_t one_long = + psch::EstimateStringArrayAttrBytes("relation_names", {"public.a_much_longer_name"}); + size_t two = psch::EstimateStringArrayAttrBytes("relation_names", {"a.b", "c.d"}); + + EXPECT_GT(one_short, empty); + EXPECT_GT(one_long, one_short); + EXPECT_GT(two, one_short); +} + +TEST(EstimateStringArrayAttrBytes, CountsPerElementFramingOverhead) { + size_t zero = psch::EstimateStringArrayAttrBytes("k", {}); + size_t one = psch::EstimateStringArrayAttrBytes("k", {"abc"}); + // Each array element is its own nested AnyValue -- adding one should cost + // more than just its own string bytes. + EXPECT_GT(one - zero, 3u); +} diff --git a/test/unit/relations_budget_test.cc b/test/unit/relations_budget_test.cc new file mode 100644 index 0000000..6768556 --- /dev/null +++ b/test/unit/relations_budget_test.cc @@ -0,0 +1,19 @@ +#include "hooks/relations_budget.h" + +#include + +TEST(RelationsWouldExceedBudget, ExactlyAtBudgetFits) { + EXPECT_FALSE(PschRelationsWouldExceedBudget(0, 10, 10)); + EXPECT_TRUE(PschRelationsWouldExceedBudget(0, 11, 10)); +} + +TEST(RelationsWouldExceedBudget, NoSeparatorByteCounted) { + // current_len=5, budget=15: entry of 10 needs exactly 5+10=15, fits -- no + // extra byte reserved for a separator, unlike the old comma-joined format. + EXPECT_FALSE(PschRelationsWouldExceedBudget(5, 10, 15)); + EXPECT_TRUE(PschRelationsWouldExceedBudget(5, 11, 15)); +} + +TEST(RelationsWouldExceedBudget, ZeroBudgetAlwaysExceeds) { + EXPECT_TRUE(PschRelationsWouldExceedBudget(0, 1, 0)); +} diff --git a/test/unit/relations_decode_test.cc b/test/unit/relations_decode_test.cc new file mode 100644 index 0000000..e09d39c --- /dev/null +++ b/test/unit/relations_decode_test.cc @@ -0,0 +1,113 @@ +#include "export/relations_decode.h" + +#include + +namespace { + +// Helper: build a blob byte-string from raw char values so tests read close +// to the wire format rather than fighting C string escaping. +std::string Blob(std::initializer_list bytes) { + return std::string(bytes.begin(), bytes.end()); +} + +} // namespace + +TEST(DecodeRelations, EmptyInputYieldsEmptyResult) { + auto out = DecodeRelations("", 0); + EXPECT_FALSE(out.truncated); + EXPECT_TRUE(out.namespaces.empty()); + EXPECT_TRUE(out.names.empty()); + EXPECT_TRUE(out.kinds.empty()); +} + +TEST(DecodeRelations, HeaderOnlyNoEntries) { + auto blob = Blob({0, 0}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + EXPECT_FALSE(out.truncated); + EXPECT_TRUE(out.namespaces.empty()); +} + +TEST(DecodeRelations, SingleEntry) { + // truncated=0, count=1, ns="public"(6), name="foo1"(4), kind='r' + auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 4, 'f', 'o', 'o', '1', 'r'}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + EXPECT_FALSE(out.truncated); + ASSERT_EQ(out.namespaces.size(), 1u); + EXPECT_EQ(out.namespaces[0], "public"); + EXPECT_EQ(out.names[0], "foo1"); + EXPECT_EQ(out.kinds[0], "r"); +} + +TEST(DecodeRelations, DotInsideNameSurvivesIntact) { + // A name containing a literal '.' must not be split -- this is the whole + // point of moving off the comma/dot-joined text format. + auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 10, 'w', 'e', 'i', 'r', + 'd', '.', 'n', 'a', 'm', 'e', 'r'}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + ASSERT_EQ(out.names.size(), 1u); + EXPECT_EQ(out.names[0], "weird.name"); +} + +TEST(DecodeRelations, CommaInsideNameSurvivesIntact) { + auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 9, 'h', 'a', 's', ',', + 'c', 'o', 'm', 'm', 'a', 'r'}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + ASSERT_EQ(out.names.size(), 1u); + EXPECT_EQ(out.names[0], "has,comma"); +} + +TEST(DecodeRelations, MultipleEntriesAndAllRelkinds) { + auto blob = Blob({ + 0, 3, + 6, 'p', 'u', 'b', 'l', 'i', 'c', 2, 'r', '1', 'r', + 6, 'p', 'u', 'b', 'l', 'i', 'c', 2, 'v', '1', 'v', + 6, 'p', 'u', 'b', 'l', 'i', 'c', 2, 'm', '1', 'm', + }); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + ASSERT_EQ(out.names.size(), 3u); + EXPECT_EQ(out.names[0], "r1"); + EXPECT_EQ(out.kinds[0], "r"); + EXPECT_EQ(out.names[1], "v1"); + EXPECT_EQ(out.kinds[1], "v"); + EXPECT_EQ(out.names[2], "m1"); + EXPECT_EQ(out.kinds[2], "m"); +} + +TEST(DecodeRelations, TruncatedFlagIsReported) { + auto blob = Blob({1, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 4, 'f', 'o', 'o', '1', 'r'}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(out.truncated); +} + +TEST(DecodeRelations, TruncatedWithZeroCountYieldsEmptyButTruncated) { + // The blob hooks.c emits when a query never got a nesting slot at all + // (table full) -- must read as truncated, not as "zero relations". + auto blob = Blob({1, 0}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(out.truncated); + EXPECT_TRUE(out.namespaces.empty()); +} + +TEST(DecodeRelations, LenShorterThanBufferOnlyReadsLenBytes) { + auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 4, 'f', 'o', 'o', '1', 'r', + 'I', 'G', 'N', 'O', 'R', 'E', 'D'}); + auto out = DecodeRelations(blob.data(), 15); // 15 == real entry length + ASSERT_EQ(out.names.size(), 1u); + EXPECT_EQ(out.names[0], "foo1"); +} + +TEST(DecodeRelations, MalformedLengthDoesNotReadOutOfBounds) { + // Declares count=1, but ns_len claims 200 bytes when only 3 remain. + // Must stop cleanly, not read/crash past the buffer. + auto blob = Blob({0, 1, 200, 'a', 'b', 'c'}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(out.names.empty()); +} + +TEST(DecodeRelations, OomSentinelReportsTruncatedNoEntries) { + // len==1 is the DSA-OOM sentinel (see relations_budget.h). + auto blob = Blob({0}); + auto out = DecodeRelations(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(out.truncated); + EXPECT_TRUE(out.namespaces.empty()); +} diff --git a/test/unit/relations_nesting_test.cc b/test/unit/relations_nesting_test.cc new file mode 100644 index 0000000..3684a87 --- /dev/null +++ b/test/unit/relations_nesting_test.cc @@ -0,0 +1,229 @@ +#include "hooks/relations_nesting.h" + +#include + +namespace { +constexpr int kMaxDepth = 16; +const int kKeys[20] = {}; // distinct addresses to use as fake QueryDesc* keys +} // namespace + +TEST(RelationsNesting, PushReservesDistinctSlotsForDistinctKeys) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int seen[kMaxDepth] = {}; + for (int i = 0; i < kMaxDepth; i++) { + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[i], 0); + ASSERT_GE(slot, 0); + seen[slot]++; + } + for (int i = 0; i < kMaxDepth; i++) { + EXPECT_EQ(seen[i], 1); + } +} + +TEST(RelationsNesting, PushPastCapacityReturnsNegativeOne) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + for (int i = 0; i < kMaxDepth; i++) { + PschRelationsNestingPush(table, kMaxDepth, &kKeys[i], 0); + } + EXPECT_EQ(PschRelationsNestingPush(table, kMaxDepth, &kKeys[kMaxDepth], 0), -1); +} + +TEST(RelationsNesting, PopReturnsPushedSlotThenFindsNothingTwice) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 0); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); +} + +TEST(RelationsNesting, PopByWrongKeyFindsNothingAndLeavesRealEntryIntact) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 0); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); +} + +TEST(RelationsNesting, OutOfOrderPopDoesNotCrossContaminate) { + // Two concurrently open cursors: opened A then B, but closed A first -- + // A's pop must not return B's slot. + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot_a = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 0); + int slot_b = PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 0); + ASSERT_NE(slot_a, slot_b); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot_a); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), slot_b); +} + +TEST(RelationsNesting, AbortReleasesOnlyEntriesAtOrAboveLevel) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot_a = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); + + PschRelationsNestingAbortTo(table, kMaxDepth, 2); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); // rolled back + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot_a); // survives +} + +TEST(RelationsNesting, AbortAtOuterLevelReleasesEverythingNestedInside) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); + + PschRelationsNestingAbortTo(table, kMaxDepth, 1); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); +} + +TEST(RelationsNesting, CommitReparentsEntryOutOfReachOfASiblingsLaterAbort) { + // A cursor opened at level 2, released (committed): without reparenting, a + // later sibling savepoint reusing level 2 would sweep it away on its abort. + PschRelationsNestingEntry table[kMaxDepth] = {}; + int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 2); + + PschRelationsNestingReparentCommitted(table, kMaxDepth, 2); + PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); + PschRelationsNestingAbortTo(table, kMaxDepth, 2); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); +} + +TEST(RelationsNesting, CommitReparentsToLevelZeroSurvivesANextTransactionsAbort) { + // A WITH HOLD cursor surviving a top-level commit: nestingLevel resets to + // 1 for the next transaction, so without reparenting to 0 that + // transaction's own abort would sweep it away too. + PschRelationsNestingEntry table[kMaxDepth] = {}; + int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + + PschRelationsNestingReparentCommitted(table, kMaxDepth, 1); + PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 1); + PschRelationsNestingAbortTo(table, kMaxDepth, 1); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); +} + +TEST(RelationsNesting, TouchIsNoOpForAnUnknownKey) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[1], 5); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); +} + +TEST(RelationsNesting, TouchNeverLowersAnAlreadyDeeperActiveLevel) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 3); + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 1); + + PschRelationsNestingAbortTo(table, kMaxDepth, 3); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); +} + +// The leak this guards against: a cursor declared at level 1, FETCHed from +// at level 2 (inside a savepoint) that fails -- created_level alone misses it. +TEST(RelationsNesting, AbortSweepsAnOlderCursorThatWasFetchedFromInsideTheAbortingSavepoint) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2); + + PschRelationsNestingAbortTo(table, kMaxDepth, 2); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); +} + +TEST(RelationsNesting, AbortAtADeeperLevelThanAnyUseLeavesTheCursorAlone) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 1); + + PschRelationsNestingAbortTo(table, kMaxDepth, 2); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); +} + +// active_level must reparent down on commit just like created_level does, +// or a later sibling savepoint reusing the same level would sweep it away. +TEST(RelationsNesting, CommitReparentsActiveLevelSoALaterSiblingAbortDoesNotOverReach) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2); + + PschRelationsNestingReparentCommitted(table, kMaxDepth, 2); + PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); + PschRelationsNestingAbortTo(table, kMaxDepth, 2); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); +} + +TEST(RelationsNesting, TouchReturnsThePreviousActiveLevel) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + + EXPECT_EQ(PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2), 1); + EXPECT_EQ(PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 3), 2); +} + +// A cursor survives an unrelated savepoint rollback after a successful FETCH +// -- Postgres only fails a portal that was still mid-execution at abort time. +TEST(RelationsNesting, SuccessfulFetchSurvivesAnUnrelatedSavepointRollback) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + + int prev = PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2); + PschRelationsNestingSetActiveLevel(table, kMaxDepth, &kKeys[0], prev); // FETCH succeeded + + PschRelationsNestingAbortTo(table, kMaxDepth, 2); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); +} + +TEST(RelationsNesting, SetActiveLevelIsNoOpForAnUnknownKey) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + + PschRelationsNestingSetActiveLevel(table, kMaxDepth, &kKeys[1], 1); + + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); +} + +// The leak this guards against: a portal Postgres fails via a path that +// never calls Touch (e.g. FETCH BACKWARD on a NO SCROLL cursor errors before +// ExecutorRun), so the abort sweep never frees it -- the slot must not leak. +TEST(RelationsNesting, PushReclaimsAnAlreadyInUseSlotForTheSameKey) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); + + EXPECT_EQ(PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 2), slot); + + // The reclaim didn't consume a second slot -- every other key still fits. + for (int i = 1; i < kMaxDepth; i++) { + EXPECT_GE(PschRelationsNestingPush(table, kMaxDepth, &kKeys[i], 0), 0); + } +} + +TEST(RelationsNesting, PushOnTheSameLeakedKeyNeverExhaustsCapacity) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + for (int i = 0; i < kMaxDepth * 2; i++) { + EXPECT_GE(PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], i), 0); + } +} + +TEST(RelationsNesting, PushReclaimDiscardsStaleLevelDataFromTheOrphanedEntry) { + PschRelationsNestingEntry table[kMaxDepth] = {}; + int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 5); + PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 9); + + EXPECT_EQ(PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1), slot); + + // A later abort at the orphan's old active_level (9) must not sweep the + // reclaimed entry -- its level data should reflect only the new reservation. + PschRelationsNestingAbortTo(table, kMaxDepth, 9); + EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); +} diff --git a/vcpkg.json b/vcpkg.json index d2720ee..bff3552 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -14,6 +14,7 @@ }, "lz4", "openssl", - "zstd" + "zstd", + "gtest" ] } From 8734538b83862811e453b5d0b5c6d588add79cc4 Mon Sep 17 00:00:00 2001 From: Aliaksei Andryiashau Date: Mon, 16 Nov 2026 15:17:54 +0100 Subject: [PATCH 2/5] fix: stop over-reporting pruned partitions, collect at ExecutorEnd Two fixes from review (jsc0218). Partition over-reporting: a generic plan's runtime partition pruning still adds every child partition's range-table entry even though only one is scanned at execution time, so a partitioned-table query reported every partition, not just the one it touched. Partition children now collapse into their parent when the parent is also in the range table -- the planner always creates it first, so the existing dedup pass catches this with no second pass. A partition queried directly, with no parent present, still reports on its own. Covered by new TAP subtests (a forced-generic-plan join, and a direct partition-child query). Collection moved from ExecutorCheckPerms_hook to ExecutorEnd_hook: ExecutorEnd already receives query_desc->plannedstmt->rtable, the same range table ExecCheckPermissions sees at executor start, so nothing needs staging between the two hooks. This removes the 16-slot nesting table, its xact/subxact callbacks, and the QueryDesc-tracking plumbing that only existed to carry the collected blob between hooks -- each nested invocation already gets its own QueryDesc, so reentrancy is free. Deletes include/hooks/relations_nesting.h and its unit tests outright. Also inlines PschRelationsWouldExceedBudget (a one-line arithmetic check with a single call site) and drops its dedicated unit test file, in line with the same simplification. Dedup/collapse set switched from a fixed Oid[1024] array to a simplehash set sized to the range table itself (review: Copilot, Cursor). A collapsed partition consumes a slot without consuming any byte-budget bytes, so the array could silently overflow on a >1024-relation plan, and its linear scan made every lookup O(n) with nothing bounding the work per execution. Covered by a new TAP subtest exceeding 1024 relations in one range table. Collapse decision switched from a relispartition()/get_partition_parent() catalog guess to plannedstmt->appendRelations (review: Copilot). The old guess collapsed any RTE whose partition parent was already seen, with no way to tell a planner-generated partition child apart from the same table named explicitly in the query, so `FROM ONLY p CROSS JOIN p_child` dropped the explicit p_child reference while the reverse FROM order kept it. AppendRelInfo only exists for RTEs the planner itself synthesized via inheritance/partition expansion, so an explicit reference is never mistaken for one, and the two catalog lookups per partition RTE are gone. Covered by a new TAP subtest querying both FROM orders. Collapse further restricted to AppendRelInfo parents that are themselves RTE_RELATION (review: Copilot). The same AppendRelInfo mechanism also flattens UNION ALL branches (pull_up_simple_union_all), so a view defined as `SELECT * FROM t1 UNION ALL SELECT * FROM t2` had its branches silently collapsed into the view's own already-seen OID, dropping t1 and t2 from the export with relations_truncated left false. A UNION ALL/view append parent stays RTE_SUBQUERY even after pull-up sets its inh flag, so requiring RTE_RELATION excludes it while leaving real partition/ inheritance parents (always RTE_RELATION) unaffected. Covered by a new TAP subtest on a UNION ALL view. Collapse check moved ahead of the seen-set insert (review: Copilot). A collapsed partition child was still being inserted into `seen` before the collapse decision ran, so its OID stayed marked seen even though it was never reported. A later genuine, unflattened reference to that same relid in the same range table -- e.g. a CROSS JOIN against an OFFSET-0 subquery wrapping the same partition -- then hit the seen-set as a false duplicate and was silently dropped, again with relations_truncated left false. Running the collapse check first means a collapsed child never touches `seen`, so the explicit reference is evaluated on its own merits. This also drops the now-redundant parent-already-seen lookup: the planner-order invariant already guarantees it holds whenever parent_rti is set and the parent is RTE_RELATION. Covered by a new TAP subtest. --- CMakeLists.txt | 2 - docs/reference/hooks.mdx | 22 +- include/hooks/relations_budget.h | 19 -- include/hooks/relations_nesting.h | 130 --------- .../20261116000001_add_relation_columns.sql | 3 +- src/config/guc.c | 4 +- src/hooks/hooks.c | 263 +++++------------- t/039_relations_export.pl | 201 ++++++++++++- test/regression/expected/relations.out | 2 +- test/regression/sql/relations.sql | 2 +- test/unit/relations_budget_test.cc | 19 -- test/unit/relations_decode_test.cc | 4 +- test/unit/relations_nesting_test.cc | 229 --------------- 13 files changed, 285 insertions(+), 615 deletions(-) delete mode 100644 include/hooks/relations_nesting.h delete mode 100644 test/unit/relations_budget_test.cc delete mode 100644 test/unit/relations_nesting_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 513d0af..4e294ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,8 +156,6 @@ if(PSCH_BUILD_UNIT_TESTS) find_package(GTest CONFIG REQUIRED) add_executable(pg_stat_ch_unittests test/unit/relations_decode_test.cc - test/unit/relations_budget_test.cc - test/unit/relations_nesting_test.cc test/unit/otel_string_array_attr_test.cc ) target_compile_features(pg_stat_ch_unittests PRIVATE cxx_std_17) diff --git a/docs/reference/hooks.mdx b/docs/reference/hooks.mdx index 9297291..5a3a052 100644 --- a/docs/reference/hooks.mdx +++ b/docs/reference/hooks.mdx @@ -10,7 +10,6 @@ pg_stat_ch uses PostgreSQL's extensibility hooks to capture query telemetry with | Hook | Purpose | When Called | |------|---------|-------------| | `ExecutorStart_hook` | Initialize instrumentation | Before query execution begins | -| `ExecutorCheckPerms_hook` | Collect touched relations | During permission checking, early in executor startup | | `ExecutorRun_hook` | Track nesting level | During query execution | | `ExecutorFinish_hook` | Track nesting level | After execution, before cleanup | | `ExecutorEnd_hook` | Capture metrics and enqueue event | After query completes | @@ -33,20 +32,6 @@ The executor hooks work together to capture metrics for optimizable queries (SEL 4. Initializes CPU time baseline via `getrusage()` 5. Enables instrumentation to collect buffer/timing stats -### ExecutorCheckPerms_hook - -**Called:** During permission checking, as part of `ExecutorStart` -- PostgreSQL hands every relation in the query's range table to this hook before deciding whether the current role may access it. - -**What pg_stat_ch does:** - -1. Skips parallel workers (the leader already captures this) -2. Walks the range table, resolving each distinct relation to its schema-qualified name and `pg_class.relkind` -3. Encodes the result into a length-prefixed binary buffer, bounded by `pg_stat_ch.relations_max_len`; queries that exceed the budget are truncated and flagged via `relations_truncated` -4. Stages the buffer in a per-nesting-level slot rather than a single shared buffer, so a nested executor invocation (a non-inlined SQL function, PL/pgSQL, dynamic `EXECUTE`, a trigger, SPI) can't clobber the outer statement's already-collected relations -5. Defers to the previous hook (or allows by default) for the actual permission decision -- this hook only observes, it never denies access - -Collection is skipped entirely (not just discarded afterward) when `pg_stat_ch.relations_enabled` is off. - ### ExecutorRun_hook **Called:** When the executor actually runs the query plan. @@ -74,7 +59,8 @@ This hook is used purely for nesting level tracking. The actual metric capture h 1. Finalizes instrumentation (`InstrEndLoop`) 2. Computes CPU time delta from `getrusage()` 3. Extracts all metrics from `QueryDesc` -4. Builds a `PschEvent` and enqueues it to shared memory +4. If `pg_stat_ch.relations_enabled` is on, walks `query_desc->plannedstmt->rtable`, resolving each distinct relation to its schema-qualified name and `pg_class.relkind` into a length-prefixed buffer bounded by `pg_stat_ch.relations_max_len` (queries that exceed the budget are truncated and flagged via `relations_truncated`) +5. Builds a `PschEvent` and enqueues it to shared memory ## Metrics Captured from QueryDesc @@ -190,9 +176,6 @@ Parallel workers execute portions of a query plan. pg_stat_ch: │ ├─ Record start time, CPU baseline │ │ └─ Enable instrumentation │ │ │ -│ ExecutorCheckPerms_hook │ -│ └─ Collect touched relations (schema, name, relkind) │ -│ │ │ ExecutorRun_hook / ExecutorFinish_hook │ │ └─ Track nesting level │ │ │ @@ -200,6 +183,7 @@ Parallel workers execute portions of a query plan. pg_stat_ch: │ ├─ Finalize instrumentation │ │ ├─ Compute CPU delta │ │ ├─ Extract metrics from QueryDesc │ +│ ├─ Collect touched relations (schema, name, relkind) │ │ └─ Enqueue PschEvent to shared memory │ │ │ │ ProcessUtility_hook │ diff --git a/include/hooks/relations_budget.h b/include/hooks/relations_budget.h index 29000e9..2d5397b 100644 --- a/include/hooks/relations_budget.h +++ b/include/hooks/relations_budget.h @@ -1,14 +1,6 @@ -// Pure C, no PostgreSQL headers -- kept separate from hooks.c so it's -// unit-testable in isolation (see test/unit/relations_budget_test.cc). #ifndef PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ #define PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ -#include - -#ifdef __cplusplus -extern "C" { -#endif - // [truncated:u8][count:u8] -- the GUC's minimum must never go below this. #define PSCH_RELATIONS_HEADER_LEN 2 @@ -16,15 +8,4 @@ extern "C" { // relations) and real blob lengths (>= PSCH_RELATIONS_HEADER_LEN). #define PSCH_RELATIONS_OOM_LEN 1 -// True if appending an entry of entry_len bytes to a buffer that already -// holds current_len bytes would exceed budget. No separator byte to -// account for -- every entry is self-delimiting (length-prefixed). -static inline bool PschRelationsWouldExceedBudget(int current_len, int entry_len, int budget) { - return (current_len + entry_len) > budget; -} - -#ifdef __cplusplus -} -#endif - #endif // PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ diff --git a/include/hooks/relations_nesting.h b/include/hooks/relations_nesting.h deleted file mode 100644 index 0314758..0000000 --- a/include/hooks/relations_nesting.h +++ /dev/null @@ -1,130 +0,0 @@ -// Pure C, no PostgreSQL headers -- kept separate from hooks.c so it's -// unit-testable in isolation (see test/unit/relations_nesting_test.cc). -#ifndef PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_NESTING_H_ -#define PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_NESTING_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// Keyed by identity (e.g. a QueryDesc*), not by call depth -- so a level -// can be released out of order (concurrently open cursors don't close in -// the order they were opened) without handing one query's buffer to -// another. `created_level` and `active_level` (deepest level used at, see -// PschRelationsNestingTouch) are opaque caller-defined tags used by -// PschRelationsNestingAbortTo. -typedef struct { - const void* key; - int created_level; - int active_level; - bool in_use; -} PschRelationsNestingEntry; - -// Returns the index of the in-use slot reserved for `key`, or -1 if none. -static inline int PschRelationsNestingFind(const PschRelationsNestingEntry* table, int max_depth, - const void* key) { - for (int i = 0; i < max_depth; i++) { - if (table[i].in_use && table[i].key == key) { - return i; - } - } - return -1; -} - -// Reserves a slot for `key`, returning its index, or -1 if all max_depth -// slots are already in use. Reclaims an existing in-use entry for the same -// key instead of adding a duplicate -- guards against a prior reservation -// for this address never having been released (see PschRelationsNestingPop). -static inline int PschRelationsNestingPush(PschRelationsNestingEntry* table, int max_depth, - const void* key, int created_level) { - int reclaim = PschRelationsNestingFind(table, max_depth, key); - if (reclaim >= 0) { - table[reclaim].created_level = created_level; - table[reclaim].active_level = created_level; - return reclaim; - } - for (int i = 0; i < max_depth; i++) { - if (!table[i].in_use) { - table[i].key = key; - table[i].created_level = created_level; - table[i].active_level = created_level; - table[i].in_use = true; - return i; - } - } - return -1; -} - -// Records `key` as used at `level`, returning its prior active_level (or -1 -// if unknown). Never lowers active_level; call SetActiveLevel to restore it. -static inline int PschRelationsNestingTouch(PschRelationsNestingEntry* table, int max_depth, - const void* key, int level) { - int i = PschRelationsNestingFind(table, max_depth, key); - if (i < 0) { - return -1; - } - int prev = table[i].active_level; - if (level > prev) { - table[i].active_level = level; - } - return prev; -} - -// Restores active_level to `level`; no-op if unknown. -static inline void PschRelationsNestingSetActiveLevel(PschRelationsNestingEntry* table, - int max_depth, const void* key, int level) { - int i = PschRelationsNestingFind(table, max_depth, key); - if (i >= 0) { - table[i].active_level = level; - } -} - -// Frees and returns the slot reserved for `key`, or -1 if none is (e.g. Push -// returned -1 for it, or PschRelationsNestingAbortTo already released it). -static inline int PschRelationsNestingPop(PschRelationsNestingEntry* table, int max_depth, - const void* key) { - int i = PschRelationsNestingFind(table, max_depth, key); - if (i >= 0) { - table[i].in_use = false; - } - return i; -} - -// Frees every slot created or used at `level` or deeper -- covers both -// rolled-back pushes and older portals PostgreSQL fails mid-abort. -static inline void PschRelationsNestingAbortTo(PschRelationsNestingEntry* table, int max_depth, - int level) { - for (int i = 0; i < max_depth; i++) { - if (table[i].in_use && - (table[i].created_level >= level || table[i].active_level >= level)) { - table[i].in_use = false; - } - } -} - -// A committing (sub)transaction's surviving entries (e.g. a cursor opened -// inside it) must move to its parent's level -- otherwise a later sibling -// subtransaction reusing this same level number would wrongly sweep them on -// its own abort. -static inline void PschRelationsNestingReparentCommitted(PschRelationsNestingEntry* table, - int max_depth, int committed_level) { - for (int i = 0; i < max_depth; i++) { - if (!table[i].in_use) { - continue; - } - if (table[i].created_level == committed_level) { - table[i].created_level = committed_level - 1; - } - if (table[i].active_level == committed_level) { - table[i].active_level = committed_level - 1; - } - } -} - -#ifdef __cplusplus -} -#endif - -#endif // PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_NESTING_H_ diff --git a/schema/migrations/20261116000001_add_relation_columns.sql b/schema/migrations/20261116000001_add_relation_columns.sql index 6393409..5061614 100644 --- a/schema/migrations/20261116000001_add_relation_columns.sql +++ b/schema/migrations/20261116000001_add_relation_columns.sql @@ -1,6 +1,5 @@ -- Adds structured relation columns: namespace, name, and relkind kept as --- separate parallel arrays. Collected in src/hooks/hooks.c --- (ExecutorCheckPerms_hook). +-- separate parallel arrays. Collected in src/hooks/hooks.c at ExecutorEnd. -- +goose Up diff --git a/src/config/guc.c b/src/config/guc.c index eeb1bb9..329c01f 100644 --- a/src/config/guc.c +++ b/src/config/guc.c @@ -430,8 +430,8 @@ void PschInitGuc(void) { DefineCustomBoolVariable( "pg_stat_ch.relations_enabled", "Collect the relations array (tables/views a query touches).", - "When enabled, the background worker resolves each query's touched " - "relations via ExecutorCheckPerms_hook and exports them as the " + "When enabled, each backend resolves a query's touched relations from " + "its plan's range table at ExecutorEnd and exports them as the " "relation_namespaces/relation_names/relation_kinds/relations_truncated " "columns. Apply the relations schema migrations before enabling. " "PGC_POSTMASTER: the ClickHouse exporter keeps its column set for the " diff --git a/src/hooks/hooks.c b/src/hooks/hooks.c index 278b4d7..76451df 100644 --- a/src/hooks/hooks.c +++ b/src/hooks/hooks.c @@ -8,6 +8,7 @@ #include "access/xact.h" #include "catalog/pg_class.h" #include "commands/dbcommands.h" +#include "common/hashfn.h" #include "common/ip.h" #include "common/pg_prng.h" #include "executor/executor.h" @@ -16,8 +17,10 @@ #include "lib/stringinfo.h" #include "libpq/libpq-be.h" #include "miscadmin.h" +#include "nodes/pathnodes.h" #include "nodes/queryjumble.h" #include "parser/analyze.h" +#include "parser/parsetree.h" #include "postmaster/bgworker.h" #include "storage/proc.h" #include "tcop/utility.h" @@ -29,10 +32,9 @@ #include "compat.h" #include "config/guc.h" #include "hooks/hooks.h" -#include "hooks/relations_budget.h" -#include "hooks/relations_nesting.h" #include "hooks/query_normalize.h" #include "hooks/query_normalize_state.h" +#include "hooks/relations_budget.h" #include "hooks/string_utils.h" #include "queue/event.h" #include "queue/shmem.h" @@ -43,7 +45,6 @@ static ExecutorStart_hook_type prev_executor_start = NULL; static ExecutorRun_hook_type prev_executor_run = NULL; static ExecutorFinish_hook_type prev_executor_finish = NULL; static ExecutorEnd_hook_type prev_executor_end = NULL; -static ExecutorCheckPerms_hook_type prev_executor_check_perms = NULL; static ProcessUtility_hook_type prev_process_utility = NULL; static emit_log_hook_type prev_emit_log_hook = NULL; @@ -68,25 +69,6 @@ static bool system_init = false; static int GetClientAddress(char* buf, int buf_size); static void ResolveNames(PschEvent* event); -// Relations collection spans two hooks: ExecutorCheckPerms_hook stages a -// query's touched tables into one of these buffers, ExecutorEnd_hook reads -// and frees it. Buffers are keyed by QueryDesc* rather than call depth, and -// freed on transaction/subtransaction abort as well as normal completion -- -// see PschRelationsNestingXactCallback and PopRelationsNestingLevel for why. -#define PSCH_RELATIONS_MAX_NESTING_DEPTH 16 -// Zero-initialized by static storage duration, so a fresh slot's .data is -// NULL until PschExecutorCheckPerms calls initStringInfo on it. -static StringInfoData relations_bufs[PSCH_RELATIONS_MAX_NESTING_DEPTH]; -static PschRelationsNestingEntry relations_nesting_table[PSCH_RELATIONS_MAX_NESTING_DEPTH]; -static bool relations_nesting_callbacks_registered = false; - -// ExecutorCheckPerms_hook only receives the range table, not the QueryDesc; -// PschExecutorStart supplies this (see CallExecutorStartChain). NULL outside -// CallExecutorStartChain's call to standard_ExecutorStart -- that's what -// distinguishes a real ExecutorCheckPerms_hook call from COPY/RI_Initial_Check -// calling it directly, with no QueryDesc to key a reservation on. -static QueryDesc* relations_active_query_desc = NULL; - // Cache for session-stable values to avoid repeated catalog lookups on every query. // Following pg_stat_monitor's pattern of caching client IP (pg_stat_monitor.c:73-96). // Database name and client address never change within a session. Username is @@ -328,58 +310,22 @@ static void CopyQueryText(PschEvent* event, uint64 query_id) { sizeof(event->query), &event->query_len); } -// Free reservations an aborting (sub)transaction never released itself; a -// WITH HOLD cursor surviving a commit is reparented to level 0, same reason -// as PschRelationsNestingReparentCommitted's subxact-level callers. -static void PschRelationsNestingXactCallback(XactEvent event, void* arg pg_attribute_unused()) { - if (event == XACT_EVENT_ABORT || event == XACT_EVENT_PARALLEL_ABORT) { - PschRelationsNestingAbortTo(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, 1); - } else if (event == XACT_EVENT_COMMIT || event == XACT_EVENT_PARALLEL_COMMIT) { - PschRelationsNestingReparentCommitted(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, 1); - } -} - -static void PschRelationsNestingSubXactCallback(SubXactEvent event, - SubTransactionId my_subid pg_attribute_unused(), - SubTransactionId parent_subid pg_attribute_unused(), - void* arg pg_attribute_unused()) { - if (event == SUBXACT_EVENT_ABORT_SUB) { - PschRelationsNestingAbortTo(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, - GetCurrentTransactionNestLevel()); - } else if (event == SUBXACT_EVENT_COMMIT_SUB) { - PschRelationsNestingReparentCommitted(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, - GetCurrentTransactionNestLevel()); - } -} - -static void EnsureRelationsNestingCallbacksRegistered(void) { - if (relations_nesting_callbacks_registered) { - return; - } - RegisterXactCallback(PschRelationsNestingXactCallback, NULL); - RegisterSubXactCallback(PschRelationsNestingSubXactCallback, NULL); - relations_nesting_callbacks_registered = true; -} - -// Must run unconditionally on every ExecutorEnd call. -static int PopRelationsNestingLevel(QueryDesc* query_desc) { - return PschRelationsNestingPop(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, - query_desc); -} - -// slot < 0 means the nesting table was genuinely full (relations_enabled=off -// still reserves a slot, see PschExecutorCheckPerms). -static void CopyRelations(PschEvent* event, int slot) { - if (slot < 0) { - event->relations[0] = 1; // truncated - event->relations[1] = 0; // count - event->relations_len = PSCH_RELATIONS_HEADER_LEN; +static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos); +static void CopyRelations(PschEvent* event, QueryDesc* query_desc) { + if (!psch_relations_enabled) { + event->relations_len = 0; return; } - Assert(relations_bufs[slot].len <= (int)sizeof(event->relations)); - memcpy(event->relations, relations_bufs[slot].data, relations_bufs[slot].len); - event->relations_len = (uint16)relations_bufs[slot].len; + MemoryContext oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + StringInfoData buf; + initStringInfo(&buf); + CollectRelations(&buf, query_desc->plannedstmt->rtable, query_desc->plannedstmt->appendRelations); + Assert(buf.len <= (int)sizeof(event->relations)); + event->relations_len = (uint16)Min(buf.len, (int)sizeof(event->relations)); + memcpy(event->relations, buf.data, event->relations_len); + pfree(buf.data); + MemoryContextSwitchTo(oldcxt); } // Resolve database and user names, using the session cache when available. @@ -449,7 +395,7 @@ static uint64 QueryDurationUs(QueryDesc* query_desc) { } static void BuildEventFromQueryDesc(QueryDesc* query_desc, PschEvent* event, int64 cpu_user_us, - int64 cpu_sys_us, int relations_slot) { + int64 cpu_sys_us) { InitBaseEvent(event, query_start_ts, current_query_is_top_level, ConvertCmdType(query_desc->operation)); event->queryid = query_desc->plannedstmt->queryId; @@ -470,7 +416,7 @@ static void BuildEventFromQueryDesc(QueryDesc* query_desc, PschEvent* event, int CopyParallelWorkerInfo(event, query_desc); CopyClientContext(event); CopyQueryText(event, query_desc->plannedstmt->queryId); - CopyRelations(event, relations_slot); + CopyRelations(event, query_desc); } // post_parse_analyze_hook — decide query text at parse time. @@ -516,28 +462,13 @@ static void PschPostParseAnalyze(ParseState* pstate, Query* query, PschJumbleSta } } -// PG_FINALLY restores both fields even if the chain errors. -static void CallExecutorStartChain(QueryDesc* query_desc, int eflags) { - QueryDesc* prev_active_query_desc = relations_active_query_desc; - relations_active_query_desc = query_desc; - PG_TRY(); - { +static void PschExecutorStart(QueryDesc* query_desc, int eflags) { + if (IsParallelWorker()) { if (prev_executor_start != NULL) { prev_executor_start(query_desc, eflags); } else { standard_ExecutorStart(query_desc, eflags); } - } - PG_FINALLY(); - { - relations_active_query_desc = prev_active_query_desc; - } - PG_END_TRY(); -} - -static void PschExecutorStart(QueryDesc* query_desc, int eflags) { - if (IsParallelWorker()) { - CallExecutorStartChain(query_desc, eflags); return; } @@ -560,7 +491,11 @@ static void PschExecutorStart(QueryDesc* query_desc, int eflags) { } #endif - CallExecutorStartChain(query_desc, eflags); + if (prev_executor_start != NULL) { + prev_executor_start(query_desc, eflags); + } else { + standard_ExecutorStart(query_desc, eflags); + } #if PG_VERSION_NUM < 190000 if (track && query_desc->totaltime == NULL) { @@ -594,12 +529,6 @@ static void PschExecutorRun(QueryDesc* query_desc, ScanDirection direction, uint return; } - // Marks this query's reservation (if any) as used at the current level -- - // unconditional like Pop, so toggling psch_enabled mid-session can't orphan it. - int prev_active_level = PschRelationsNestingTouch(relations_nesting_table, - PSCH_RELATIONS_MAX_NESTING_DEPTH, query_desc, - GetCurrentTransactionNestLevel()); - nesting_level++; PG_TRY(); { @@ -616,11 +545,6 @@ static void PschExecutorRun(QueryDesc* query_desc, ScanDirection direction, uint standard_ExecutorRun(query_desc, direction, count, execute_once); } #endif - // Only on success -- an error skips this, leaving active_level raised. - if (prev_active_level >= 0) { - PschRelationsNestingSetActiveLevel(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, - query_desc, prev_active_level); - } } PG_FINALLY(); { nesting_level--; } @@ -652,9 +576,6 @@ static void PschExecutorFinish(QueryDesc* query_desc) { } static void PschExecutorEnd(QueryDesc* query_desc) { - // Unconditional: must run even on paths below that skip the export itself. - int relations_slot = PopRelationsNestingLevel(query_desc); - if (!psch_enabled || IsParallelWorker() || query_desc->plannedstmt->queryId == UINT64CONST(0)) { if (prev_executor_end != NULL) { prev_executor_end(query_desc); @@ -692,7 +613,7 @@ static void PschExecutorEnd(QueryDesc* query_desc) { } PschEvent event; - BuildEventFromQueryDesc(query_desc, &event, cpu_user_us, cpu_sys_us, relations_slot); + BuildEventFromQueryDesc(query_desc, &event, cpu_user_us, cpu_sys_us); PschEnqueueEvent(&event); if (prev_executor_end != NULL) { @@ -942,39 +863,53 @@ bool PschSuppressErrorCapture(bool suppress) { return prev; } -// True if relid is already in seen[0..nseen). -static bool AlreadySeen(const Oid* seen, int nseen, Oid relid) { - for (int i = 0; i < nseen; i++) { - if (seen[i] == relid) { - return true; - } - } - return false; -} - -// Builds the relations blob: [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count. -static void CollectRelations(StringInfo buf, List* rt) { - // Sized to PSCH_MAX_RELATIONS_LEN, not an arbitrary smaller constant: every - // entry consumes at least 1 byte of the relations_buf budget, so nseen can - // never exceed the byte budget (<= PSCH_MAX_RELATIONS_LEN - 1) before the - // WouldExceedBudget break below fires. A smaller cap here would let nseen - // saturate while relations_buf still has room, silently disabling dedup - // for later relations without disabling appends -- so a table referenced - // twice past that point would show up twice. - Oid seen[PSCH_MAX_RELATIONS_LEN]; - int nseen = 0; +// Oid set for CollectRelations' dedup/collapse below. Created fresh per call +// in the caller's es_query_cxt -- sized to the range table itself, not the +// relations_buf byte budget, since a collapsed partition consumes a slot +// here without consuming any budget bytes. +typedef struct PschRelidSeenEntry { + Oid relid; + char status; +} PschRelidSeenEntry; + +#define SH_PREFIX psch_relid_seen +#define SH_ELEMENT_TYPE PschRelidSeenEntry +#define SH_KEY_TYPE Oid +#define SH_KEY relid +#define SH_HASH_KEY(tb, key) murmurhash32((uint32)(key)) +#define SH_EQUAL(tb, a, b) ((a) == (b)) +#define SH_SCOPE static inline +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +// Builds the relations blob: +// [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count. +static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos) { + int nrt = list_length(rt); + psch_relid_seen_hash* seen = psch_relid_seen_create(CurrentMemoryContext, Max(nrt, 1), NULL); uint8 count = 0; bool truncated = false; + int rti = 0; ListCell* lc; // Defense in depth -- the GUC's own min bound (guc.c) already enforces this. Assert(psch_relations_max_len >= PSCH_RELATIONS_HEADER_LEN); + Index* parent_of_rti = palloc0(sizeof(Index) * (nrt + 1)); + ListCell* alc; + foreach (alc, append_rel_infos) { + AppendRelInfo* appinfo = (AppendRelInfo*)lfirst(alc); + Assert(appinfo->child_relid >= 1 && (int)appinfo->child_relid <= nrt); + parent_of_rti[appinfo->child_relid] = appinfo->parent_relid; + } + // Reserve the header now; patched with real values once known. appendStringInfoChar(buf, 0); appendStringInfoChar(buf, 0); foreach (lc, rt) { + rti++; RangeTblEntry* rte = (RangeTblEntry*)lfirst(lc); // Plain tables, plus view-expansion artifacts: the rewriter turns a @@ -982,19 +917,24 @@ static void CollectRelations(StringInfo buf, List* rt) { // stamps relkind='v' on that entry so it's still identifiable as a view. // RTE_RELATION alone already covers every other relkind (matview, // partitioned, foreign table) uniformly -- unaffected by that work. - // Partition scans surface the scanned child partitions here, not the - // declared parent -- that's the planner's own range table, not this. + // Partition scans surface both the parent and every child the planner + // considered, including ones pruned only at execution time. if (rte->rtekind != RTE_RELATION && rte->relkind != RELKIND_VIEW) { continue; } - if (AlreadySeen(seen, nseen, rte->relid)) { + // Collapse planner-generated children before deduplicating relation OIDs, + // so a later explicit reference to the same partition is still reported. + Index parent_rti = parent_of_rti[rti]; + if (parent_rti != 0 && rt_fetch(parent_rti, rt)->rtekind == RTE_RELATION) { continue; } - if (nseen < (int)(lengthof(seen))) { - seen[nseen++] = rte->relid; + + bool found; + psch_relid_seen_insert(seen, rte->relid, &found); + if (found) { + continue; } - Assert(nseen <= (int)(lengthof(seen))); const char* namespace_name = get_namespace_name(get_rel_namespace(rte->relid)); const char* relation_name = get_rel_name(rte->relid); @@ -1009,8 +949,9 @@ static void CollectRelations(StringInfo buf, List* rt) { Assert(ns_len < NAMEDATALEN); Assert(name_len < NAMEDATALEN); + // No separator byte to account for -- every entry is self-delimiting. int entry_len = (int)(1 + ns_len + 1 + name_len + 1); - if (PschRelationsWouldExceedBudget(buf->len, entry_len, psch_relations_max_len)) { + if (buf->len + entry_len > psch_relations_max_len) { truncated = true; break; // Budget exhausted -- truncate, same as query text does. } @@ -1025,56 +966,7 @@ static void CollectRelations(StringInfo buf, List* rt) { buf->data[0] = truncated ? 1 : 0; buf->data[1] = (char)count; -} - -// Releases the slot on query-context teardown -- catches portal failures -// that happen before ExecutorRun, which level-based sweeps can miss. -typedef struct PschRelationsNestingReleaseCallback { - MemoryContextCallback cb; - const void* key; -} PschRelationsNestingReleaseCallback; - -static void PschRelationsNestingReleaseOnContextTeardown(void* arg) { - const PschRelationsNestingReleaseCallback* release_cb = (const PschRelationsNestingReleaseCallback*)arg; - PschRelationsNestingPop(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, release_cb->key); -} - -// ExecutorCheckPerms_hook -- collects relations, then defers the actual -// permission decision to the previous hook (or the standard "allow" default). -static bool PschExecutorCheckPerms(List* rt, List* rp, bool ereport_on_violation) { - // Reservation is gated on relations_active_query_desc only, not - // relations_enabled, which instead decides whether to actually collect. - if (psch_enabled && !IsParallelWorker() && relations_active_query_desc != NULL) { - EnsureRelationsNestingCallbacksRegistered(); - int slot = PschRelationsNestingPush(relations_nesting_table, PSCH_RELATIONS_MAX_NESTING_DEPTH, - relations_active_query_desc, - GetCurrentTransactionNestLevel()); - if (slot >= 0) { - PschRelationsNestingReleaseCallback* release_cb = - MemoryContextAlloc(CurrentMemoryContext, sizeof(*release_cb)); - release_cb->cb.func = PschRelationsNestingReleaseOnContextTeardown; - release_cb->cb.arg = release_cb; - release_cb->key = relations_active_query_desc; - MemoryContextRegisterResetCallback(CurrentMemoryContext, &release_cb->cb); - - if (relations_bufs[slot].data == NULL) { - MemoryContext oldcxt = MemoryContextSwitchTo(TopMemoryContext); - initStringInfo(&relations_bufs[slot]); - MemoryContextSwitchTo(oldcxt); - } else { - resetStringInfo(&relations_bufs[slot]); - } - - if (psch_relations_enabled) { - CollectRelations(&relations_bufs[slot], rt); - } - } - } - - if (prev_executor_check_perms != NULL) { - return prev_executor_check_perms(rt, rp, ereport_on_violation); - } - return true; + psch_relid_seen_destroy(seen); } void PschInstallHooks(void) { @@ -1095,9 +987,6 @@ void PschInstallHooks(void) { prev_executor_end = ExecutorEnd_hook; ExecutorEnd_hook = PschExecutorEnd; - prev_executor_check_perms = ExecutorCheckPerms_hook; - ExecutorCheckPerms_hook = PschExecutorCheckPerms; - prev_process_utility = ProcessUtility_hook; ProcessUtility_hook = PschProcessUtility; diff --git a/t/039_relations_export.pl b/t/039_relations_export.pl index bf75e30..f628eda 100644 --- a/t/039_relations_export.pl +++ b/t/039_relations_export.pl @@ -129,6 +129,28 @@ like($names, qr/'foo1'/, "underlying base table also present (got: $names)"); }; +subtest 'UNION ALL view reports both branches alongside the view' => sub { + $node->safe_psql('postgres', 'CREATE TABLE ua1(a int)'); + $node->safe_psql('postgres', 'CREATE TABLE ua2(a int)'); + $node->safe_psql('postgres', + 'CREATE VIEW v_union_all AS SELECT * FROM ua1 UNION ALL SELECT * FROM ua2'); + + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM v_union_all'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM v_union_all%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'v_union_all'/, "view itself present (got: $names)"); + like($names, qr/'ua1'/, "first UNION ALL branch's table present (got: $names)"); + like($names, qr/'ua2'/, "second UNION ALL branch's table present (got: $names)"); +}; + subtest 'truncation over budget is flagged and does not error' => sub { psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); psch_reset_stats($node); @@ -320,6 +342,182 @@ } }; +subtest 'runtime-pruned partition children collapse into the parent' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE gp_part(id int, val text) PARTITION BY RANGE (id)'); + for my $i (0 .. 4) { + my $lo = $i * 100; + my $hi = $lo + 100; + $node->safe_psql('postgres', + "CREATE TABLE gp_part_$i PARTITION OF gp_part FOR VALUES FROM ($lo) TO ($hi)"); + } + $node->safe_psql('postgres', 'CREATE TABLE gp_other(id int)'); + + $node->safe_psql('postgres', <<'SQL'); +SET plan_cache_mode = force_generic_plan; +PREPARE gp_q(int) AS SELECT count(*) FROM gp_part JOIN gp_other USING (id) WHERE gp_part.id = $1; +EXECUTE gp_q(5); +SQL + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE '%gp_part%gp_other%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'gp_part'/, "parent reported (got: $names)"); + like($names, qr/'gp_other'/, "unrelated joined table reported (got: $names)"); + unlike($names, qr/'gp_part_\d'/, "pruned children not reported (got: $names)"); +}; + +subtest 'multilevel (subpartitioned) hierarchy collapses leaves into the root' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE sp_root(id int, val text) PARTITION BY RANGE (id)'); + $node->safe_psql('postgres', + 'CREATE TABLE sp_mid PARTITION OF sp_root FOR VALUES FROM (0) TO (1000) PARTITION BY RANGE (id)'); + for my $i (0 .. 4) { + my $lo = $i * 100; + my $hi = $lo + 100; + $node->safe_psql('postgres', + "CREATE TABLE sp_leaf_$i PARTITION OF sp_mid FOR VALUES FROM ($lo) TO ($hi)"); + } + $node->safe_psql('postgres', 'CREATE TABLE sp_other(id int)'); + + $node->safe_psql('postgres', <<'SQL'); +SET plan_cache_mode = force_generic_plan; +PREPARE sp_q(int) AS SELECT count(*) FROM sp_root JOIN sp_other USING (id) WHERE sp_root.id = $1; +EXECUTE sp_q(5); +SQL + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE '%sp_root%sp_other%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'sp_root'/, "root reported (got: $names)"); + like($names, qr/'sp_other'/, "unrelated joined table reported (got: $names)"); + unlike($names, qr/'sp_mid'/, "subpartitioned intermediate not reported (got: $names)"); + unlike($names, qr/'sp_leaf_\d'/, "leaves two levels down not reported (got: $names)"); +}; + +subtest 'subpartitioned hierarchy past the old 1024-relation seen-set cap collapses correctly' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # 600 (mid, leaf) pairs plus the root exceed PSCH_MAX_RELATIONS_LEN (1024) + # distinct relations in one range table -- regression test for the + # fixed-size Oid[1024] `seen` array this used to overflow into. + $node->safe_psql('postgres', 'CREATE TABLE big_root(id int, val text) PARTITION BY RANGE (id)'); + $node->safe_psql('postgres', <<'SQL'); +DO $do$ +DECLARE + i int; + lo int; + hi int; +BEGIN + FOR i IN 0..599 LOOP + lo := i * 20; + hi := lo + 20; + EXECUTE format( + 'CREATE TABLE big_mid_%s PARTITION OF big_root FOR VALUES FROM (%s) TO (%s) PARTITION BY RANGE (id)', + i, lo, hi); + EXECUTE format( + 'CREATE TABLE big_leaf_%s PARTITION OF big_mid_%s FOR VALUES FROM (%s) TO (%s)', + i, i, lo, hi); + END LOOP; +END +$do$; +SQL + $node->safe_psql('postgres', 'CREATE TABLE big_other(id int)'); + + $node->safe_psql('postgres', <<'SQL'); +SET plan_cache_mode = force_generic_plan; +PREPARE big_q(int) AS SELECT count(*) FROM big_root JOIN big_other USING (id) WHERE big_root.id = $1; +EXECUTE big_q(11999); +SQL + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE '%big_root%big_other%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'big_root'/, "root reported (got: $names)"); + like($names, qr/'big_other'/, "unrelated joined table reported (got: $names)"); + unlike($names, qr/'big_mid_599'/, "subpartitioned intermediate past the old cap not reported (got: $names)"); + unlike($names, qr/'big_leaf_599'/, "leaf past the old cap not reported (got: $names)"); +}; + +subtest 'partition queried directly, with no parent in the range table, still reports' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'SELECT * FROM gp_part_0'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM gp_part_0%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'gp_part_0'/, "directly-queried partition still reported (got: $names)"); +}; + +subtest 'partition named explicitly alongside its parent is reported regardless of FROM order' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', + 'SELECT * FROM ONLY gp_part CROSS JOIN gp_part_0 /* parent_first */'); + $node->safe_psql('postgres', + 'SELECT * FROM gp_part_0 CROSS JOIN ONLY gp_part /* child_first */'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $parent_first = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE '%parent_first%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($parent_first, qr/'gp_part'/, "parent reported, parent-first order (got: $parent_first)"); + like($parent_first, qr/'gp_part_0'/, + "explicit child reported, parent-first order (got: $parent_first)"); + + my $child_first = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE '%child_first%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($child_first, qr/'gp_part'/, "parent reported, child-first order (got: $child_first)"); + like($child_first, qr/'gp_part_0'/, + "explicit child reported, child-first order (got: $child_first)"); +}; + +subtest 'partition collapsed in one branch is still reported via an unflattened explicit reference' => + sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # The OFFSET 0 subquery blocks flattening, so gp_part_0 gets its own + # real RTE here, distinct from the one the gp_part scan's partition + # expansion collapses into gp_part in the same range table. + $node->safe_psql('postgres', + 'SELECT * FROM gp_part CROSS JOIN (SELECT * FROM gp_part_0 OFFSET 0) c'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE '%gp_part CROSS JOIN%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'gp_part'/, "parent reported (got: $names)"); + like($names, qr/'gp_part_0'/, "explicitly-referenced partition still reported (got: $names)"); +}; + subtest 'maximum-length identifier round-trips intact' => sub { psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); psch_reset_stats($node); @@ -691,8 +889,7 @@ $node->safe_psql('postgres', 'CREATE TABLE cursor_leak_t(a int)'); # 20 distinct failed-cursor/savepoint sequences used to exhaust all 16 - # nesting slots; must run in one session (psql, on_error_stop => 0), - # since relations_nesting_table is per-backend state. + # nesting slots; must run in one session (psql, on_error_stop => 0). my $sql = "BEGIN;\n"; for my $i (1 .. 20) { $sql .= "DECLARE cursor_leak_cur_$i NO SCROLL CURSOR FOR SELECT * FROM cursor_leak_t;\n"; diff --git a/test/regression/expected/relations.out b/test/regression/expected/relations.out index 1b2b38d..9686f10 100644 --- a/test/regression/expected/relations.out +++ b/test/regression/expected/relations.out @@ -2,7 +2,7 @@ -- Actual captured VALUES (which relations landed in the array) can only be -- verified against ClickHouse -- see t/039_relations_export.pl. This -- regression test covers what pg_regress *can* check synchronously: the GUC, --- and that ExecutorCheckPerms_hook doesn't crash/error across the basic +-- and that collection at ExecutorEnd doesn't crash/error across the basic -- shapes a per-query relations capture needs to handle (plain tables, -- schema-qualified tables, self-joins, views). CREATE EXTENSION pg_stat_ch; diff --git a/test/regression/sql/relations.sql b/test/regression/sql/relations.sql index ac13296..4458ebe 100644 --- a/test/regression/sql/relations.sql +++ b/test/regression/sql/relations.sql @@ -2,7 +2,7 @@ -- Actual captured VALUES (which relations landed in the array) can only be -- verified against ClickHouse -- see t/039_relations_export.pl. This -- regression test covers what pg_regress *can* check synchronously: the GUC, --- and that ExecutorCheckPerms_hook doesn't crash/error across the basic +-- and that collection at ExecutorEnd doesn't crash/error across the basic -- shapes a per-query relations capture needs to handle (plain tables, -- schema-qualified tables, self-joins, views). CREATE EXTENSION pg_stat_ch; diff --git a/test/unit/relations_budget_test.cc b/test/unit/relations_budget_test.cc deleted file mode 100644 index 6768556..0000000 --- a/test/unit/relations_budget_test.cc +++ /dev/null @@ -1,19 +0,0 @@ -#include "hooks/relations_budget.h" - -#include - -TEST(RelationsWouldExceedBudget, ExactlyAtBudgetFits) { - EXPECT_FALSE(PschRelationsWouldExceedBudget(0, 10, 10)); - EXPECT_TRUE(PschRelationsWouldExceedBudget(0, 11, 10)); -} - -TEST(RelationsWouldExceedBudget, NoSeparatorByteCounted) { - // current_len=5, budget=15: entry of 10 needs exactly 5+10=15, fits -- no - // extra byte reserved for a separator, unlike the old comma-joined format. - EXPECT_FALSE(PschRelationsWouldExceedBudget(5, 10, 15)); - EXPECT_TRUE(PschRelationsWouldExceedBudget(5, 11, 15)); -} - -TEST(RelationsWouldExceedBudget, ZeroBudgetAlwaysExceeds) { - EXPECT_TRUE(PschRelationsWouldExceedBudget(0, 1, 0)); -} diff --git a/test/unit/relations_decode_test.cc b/test/unit/relations_decode_test.cc index e09d39c..52c0a82 100644 --- a/test/unit/relations_decode_test.cc +++ b/test/unit/relations_decode_test.cc @@ -80,8 +80,8 @@ TEST(DecodeRelations, TruncatedFlagIsReported) { } TEST(DecodeRelations, TruncatedWithZeroCountYieldsEmptyButTruncated) { - // The blob hooks.c emits when a query never got a nesting slot at all - // (table full) -- must read as truncated, not as "zero relations". + // The blob CollectRelations emits when the budget is too small to fit even + // one entry -- must read as truncated, not as "zero relations". auto blob = Blob({1, 0}); auto out = DecodeRelations(blob.data(), static_cast(blob.size())); EXPECT_TRUE(out.truncated); diff --git a/test/unit/relations_nesting_test.cc b/test/unit/relations_nesting_test.cc deleted file mode 100644 index 3684a87..0000000 --- a/test/unit/relations_nesting_test.cc +++ /dev/null @@ -1,229 +0,0 @@ -#include "hooks/relations_nesting.h" - -#include - -namespace { -constexpr int kMaxDepth = 16; -const int kKeys[20] = {}; // distinct addresses to use as fake QueryDesc* keys -} // namespace - -TEST(RelationsNesting, PushReservesDistinctSlotsForDistinctKeys) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int seen[kMaxDepth] = {}; - for (int i = 0; i < kMaxDepth; i++) { - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[i], 0); - ASSERT_GE(slot, 0); - seen[slot]++; - } - for (int i = 0; i < kMaxDepth; i++) { - EXPECT_EQ(seen[i], 1); - } -} - -TEST(RelationsNesting, PushPastCapacityReturnsNegativeOne) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - for (int i = 0; i < kMaxDepth; i++) { - PschRelationsNestingPush(table, kMaxDepth, &kKeys[i], 0); - } - EXPECT_EQ(PschRelationsNestingPush(table, kMaxDepth, &kKeys[kMaxDepth], 0), -1); -} - -TEST(RelationsNesting, PopReturnsPushedSlotThenFindsNothingTwice) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 0); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); -} - -TEST(RelationsNesting, PopByWrongKeyFindsNothingAndLeavesRealEntryIntact) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 0); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); -} - -TEST(RelationsNesting, OutOfOrderPopDoesNotCrossContaminate) { - // Two concurrently open cursors: opened A then B, but closed A first -- - // A's pop must not return B's slot. - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot_a = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 0); - int slot_b = PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 0); - ASSERT_NE(slot_a, slot_b); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot_a); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), slot_b); -} - -TEST(RelationsNesting, AbortReleasesOnlyEntriesAtOrAboveLevel) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot_a = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); - - PschRelationsNestingAbortTo(table, kMaxDepth, 2); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); // rolled back - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot_a); // survives -} - -TEST(RelationsNesting, AbortAtOuterLevelReleasesEverythingNestedInside) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); - - PschRelationsNestingAbortTo(table, kMaxDepth, 1); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); -} - -TEST(RelationsNesting, CommitReparentsEntryOutOfReachOfASiblingsLaterAbort) { - // A cursor opened at level 2, released (committed): without reparenting, a - // later sibling savepoint reusing level 2 would sweep it away on its abort. - PschRelationsNestingEntry table[kMaxDepth] = {}; - int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 2); - - PschRelationsNestingReparentCommitted(table, kMaxDepth, 2); - PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); - PschRelationsNestingAbortTo(table, kMaxDepth, 2); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); -} - -TEST(RelationsNesting, CommitReparentsToLevelZeroSurvivesANextTransactionsAbort) { - // A WITH HOLD cursor surviving a top-level commit: nestingLevel resets to - // 1 for the next transaction, so without reparenting to 0 that - // transaction's own abort would sweep it away too. - PschRelationsNestingEntry table[kMaxDepth] = {}; - int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - - PschRelationsNestingReparentCommitted(table, kMaxDepth, 1); - PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 1); - PschRelationsNestingAbortTo(table, kMaxDepth, 1); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); -} - -TEST(RelationsNesting, TouchIsNoOpForAnUnknownKey) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[1], 5); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); -} - -TEST(RelationsNesting, TouchNeverLowersAnAlreadyDeeperActiveLevel) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 3); - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 1); - - PschRelationsNestingAbortTo(table, kMaxDepth, 3); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); -} - -// The leak this guards against: a cursor declared at level 1, FETCHed from -// at level 2 (inside a savepoint) that fails -- created_level alone misses it. -TEST(RelationsNesting, AbortSweepsAnOlderCursorThatWasFetchedFromInsideTheAbortingSavepoint) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2); - - PschRelationsNestingAbortTo(table, kMaxDepth, 2); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), -1); -} - -TEST(RelationsNesting, AbortAtADeeperLevelThanAnyUseLeavesTheCursorAlone) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 1); - - PschRelationsNestingAbortTo(table, kMaxDepth, 2); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); -} - -// active_level must reparent down on commit just like created_level does, -// or a later sibling savepoint reusing the same level would sweep it away. -TEST(RelationsNesting, CommitReparentsActiveLevelSoALaterSiblingAbortDoesNotOverReach) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2); - - PschRelationsNestingReparentCommitted(table, kMaxDepth, 2); - PschRelationsNestingPush(table, kMaxDepth, &kKeys[1], 2); - PschRelationsNestingAbortTo(table, kMaxDepth, 2); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[1]), -1); -} - -TEST(RelationsNesting, TouchReturnsThePreviousActiveLevel) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - - EXPECT_EQ(PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2), 1); - EXPECT_EQ(PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 3), 2); -} - -// A cursor survives an unrelated savepoint rollback after a successful FETCH -// -- Postgres only fails a portal that was still mid-execution at abort time. -TEST(RelationsNesting, SuccessfulFetchSurvivesAnUnrelatedSavepointRollback) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int cursor_slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - - int prev = PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 2); - PschRelationsNestingSetActiveLevel(table, kMaxDepth, &kKeys[0], prev); // FETCH succeeded - - PschRelationsNestingAbortTo(table, kMaxDepth, 2); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), cursor_slot); -} - -TEST(RelationsNesting, SetActiveLevelIsNoOpForAnUnknownKey) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - - PschRelationsNestingSetActiveLevel(table, kMaxDepth, &kKeys[1], 1); - - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); -} - -// The leak this guards against: a portal Postgres fails via a path that -// never calls Touch (e.g. FETCH BACKWARD on a NO SCROLL cursor errors before -// ExecutorRun), so the abort sweep never frees it -- the slot must not leak. -TEST(RelationsNesting, PushReclaimsAnAlreadyInUseSlotForTheSameKey) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1); - - EXPECT_EQ(PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 2), slot); - - // The reclaim didn't consume a second slot -- every other key still fits. - for (int i = 1; i < kMaxDepth; i++) { - EXPECT_GE(PschRelationsNestingPush(table, kMaxDepth, &kKeys[i], 0), 0); - } -} - -TEST(RelationsNesting, PushOnTheSameLeakedKeyNeverExhaustsCapacity) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - for (int i = 0; i < kMaxDepth * 2; i++) { - EXPECT_GE(PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], i), 0); - } -} - -TEST(RelationsNesting, PushReclaimDiscardsStaleLevelDataFromTheOrphanedEntry) { - PschRelationsNestingEntry table[kMaxDepth] = {}; - int slot = PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 5); - PschRelationsNestingTouch(table, kMaxDepth, &kKeys[0], 9); - - EXPECT_EQ(PschRelationsNestingPush(table, kMaxDepth, &kKeys[0], 1), slot); - - // A later abort at the orphan's old active_level (9) must not sweep the - // reclaimed entry -- its level data should reflect only the new reservation. - PschRelationsNestingAbortTo(table, kMaxDepth, 9); - EXPECT_EQ(PschRelationsNestingPop(table, kMaxDepth, &kKeys[0]), slot); -} From 2e71af58019a63ab3691502958819953b03117fc Mon Sep 17 00:00:00 2001 From: Aliaksei Andryiashau Date: Tue, 17 Nov 2026 13:46:09 +0100 Subject: [PATCH 3/5] fix: address review comments on relations export (docs, comments, GTest removal) - Drop the stale per-query cost figure from configuration.mdx (measured before the ExecutorEnd refactor, now stale). - Update hooks.c comments: partition/inheritance collapse no longer lists children, and document that error events carry no relations (collected only at ExecutorEnd). - events-schema.mdx: clarify relation_namespaces wording for the same two points. - Remove the GoogleTest unit tests and the vcpkg/CMake machinery behind them, following #80's precedent, including the now-dead "Run unit tests" CI step. Normal-path decoding is already proven end-to-end by t/039; add a TAP subtest there for the previously-untested failed-query path (relations_len=0). --- .github/workflows/ci.yml | 3 - CMakeLists.txt | 21 ---- docs/reference/configuration.mdx | 2 +- docs/reference/events-schema.mdx | 2 +- src/hooks/hooks.c | 10 +- t/039_relations_export.pl | 151 +++++------------------ test/unit/otel_string_array_attr_test.cc | 61 --------- test/unit/relations_decode_test.cc | 113 ----------------- vcpkg.json | 3 +- 9 files changed, 41 insertions(+), 325 deletions(-) delete mode 100644 test/unit/otel_string_array_attr_test.cc delete mode 100644 test/unit/relations_decode_test.cc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db99642..a7090f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,9 +72,6 @@ jobs: - name: Build run: cmake --build build --parallel - - name: Run unit tests - run: ctest --test-dir build --output-on-failure - - name: Install extension run: sudo cmake --install build diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e294ee..0ea3957 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,24 +145,3 @@ install(TARGETS pg_stat_ch LIBRARY DESTINATION ${PostgreSQLServer_PKGLIB_DIR}) install(FILES pg_stat_ch.control DESTINATION ${PostgreSQLServer_SHARE_DIR}/extension) file(GLOB SQL_FILES sql/pg_stat_ch--*.sql) install(FILES ${SQL_FILES} DESTINATION ${PostgreSQLServer_SHARE_DIR}/extension) - -# Unit tests for plain C++ helpers that don't need a running Postgres (see -# test/unit/). Separate from the pg_regress/TAP suites, which do. Off by -# default for image builds that COPY a .dockerignore-filtered context -# without test/ -- those only need the pg_stat_ch target, never this one. -option(PSCH_BUILD_UNIT_TESTS "Build the GoogleTest C++ unit tests" ON) -if(PSCH_BUILD_UNIT_TESTS) - enable_testing() - find_package(GTest CONFIG REQUIRED) - add_executable(pg_stat_ch_unittests - test/unit/relations_decode_test.cc - test/unit/otel_string_array_attr_test.cc - ) - target_compile_features(pg_stat_ch_unittests PRIVATE cxx_std_17) - target_include_directories(pg_stat_ch_unittests PRIVATE src include) - target_link_libraries(pg_stat_ch_unittests PRIVATE GTest::gtest GTest::gtest_main - opentelemetry-cpp::otlp_grpc_log_record_exporter) - include(GoogleTest) - gtest_discover_tests(pg_stat_ch_unittests) -endif() - diff --git a/docs/reference/configuration.mdx b/docs/reference/configuration.mdx index 24c3f00..851fda0 100644 --- a/docs/reference/configuration.mdx +++ b/docs/reference/configuration.mdx @@ -153,7 +153,7 @@ Collect the relations array (tables/views a query touches). | **Default** | `off` | | **Context** | postmaster | -When enabled, pg_stat_ch resolves each query's touched relations (tables, views, materialized views, foreign tables, partitioned tables) and exports them as three parallel arrays -- `relation_namespaces`/`relation_names`/`relation_kinds` -- plus a `relations_truncated` flag. Toggling this off skips the catalog lookups entirely, not just the storage -- useful for isolating this feature's hot-path cost from pg_stat_ch's baseline overhead. In a synthetic benchmark across join widths of 1/5/10/20 tables, the added CPU cost measured between +0.05us and +1.75us per query. +When enabled, pg_stat_ch resolves each query's touched relations (tables, views, materialized views, foreign tables, partitioned tables) and exports them as three parallel arrays -- `relation_namespaces`/`relation_names`/`relation_kinds` -- plus a `relations_truncated` flag. Toggling this off skips the catalog lookups entirely, not just the storage -- useful for isolating this feature's hot-path cost from pg_stat_ch's baseline overhead. PGC_POSTMASTER: the ClickHouse exporter keeps its column set for the life of the bgworker, so changing this requires a restart. Apply the relations schema migrations before enabling it. diff --git a/docs/reference/events-schema.mdx b/docs/reference/events-schema.mdx index d78182e..dacd2da 100644 --- a/docs/reference/events-schema.mdx +++ b/docs/reference/events-schema.mdx @@ -48,7 +48,7 @@ Label values are exported as written by the application. Don't put PII, tokens, | Column | Type | Description | |---|---|---| -| `relation_namespaces` | `Array(LowCardinality(String))` | Schema name of each distinct table/view/materialized view/foreign table/partitioned table the query touched, positionally parallel to `relation_names` and `relation_kinds`. | +| `relation_namespaces` | `Array(LowCardinality(String))` | Schema name of each distinct table/view/materialized view/foreign table/partitioned table the query touched, positionally parallel to `relation_names` and `relation_kinds`. Partitions and inheritance children are reported through their parent, not individually (for example, `SELECT * FROM inh_p` lists only `inh_p`). Recorded only for queries that complete; error events have empty relation arrays. | | `relation_names` | `Array(LowCardinality(String))` | Relation name of each distinct relation the query touched. | | `relation_kinds` | `Array(FixedString(1))` | `pg_class.relkind` for each entry: `r` (table), `v` (view), `m` (materialized view), `f` (foreign table), `p` (partitioned table), etc. | | `relations_truncated` | `Bool` | `true` if the query touched more relations than fit within `pg_stat_ch.relations_max_len`; the arrays above are incomplete for that event. | diff --git a/src/hooks/hooks.c b/src/hooks/hooks.c index 76451df..7efdde3 100644 --- a/src/hooks/hooks.c +++ b/src/hooks/hooks.c @@ -311,6 +311,12 @@ static void CopyQueryText(PschEvent* event, uint64 query_id) { } static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos); +// Relations are collected only here, at ExecutorEnd, so only queries that run +// to completion carry them. A query that errors out (including failures before +// execution starts, e.g. permission denied) never reaches ExecutorEnd, and its +// error event (built from ErrorData in CaptureLogEvent) has relations_len = 0. +// That reads as empty arrays with relations_truncated = false, so an empty list +// on an error event means "not recorded", not "touched nothing". static void CopyRelations(PschEvent* event, QueryDesc* query_desc) { if (!psch_relations_enabled) { event->relations_len = 0; @@ -917,8 +923,8 @@ static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos) { // stamps relkind='v' on that entry so it's still identifiable as a view. // RTE_RELATION alone already covers every other relkind (matview, // partitioned, foreign table) uniformly -- unaffected by that work. - // Partition scans surface both the parent and every child the planner - // considered, including ones pruned only at execution time. + // Partition and inheritance children are collapsed into their parent + // below: the parent is reported, planner-generated children are not. if (rte->rtekind != RTE_RELATION && rte->relkind != RELKIND_VIEW) { continue; } diff --git a/t/039_relations_export.pl b/t/039_relations_export.pl index f628eda..060c447 100644 --- a/t/039_relations_export.pl +++ b/t/039_relations_export.pl @@ -251,6 +251,32 @@ $node->restart(); }; +subtest 'failed query reports empty, non-truncated relations' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + # Division by zero fails during planning (constant-folded), well before + # ExecutorEnd (where CopyRelations runs) is ever reached, so the error + # event must carry no relations. Error events carry no query_text (see + # CaptureLogEvent), so this is matched by SQLSTATE instead. + $node->psql('postgres', 'SELECT * FROM foo1 WHERE 1/0 = 1', on_error_stop => 0); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw " . + "WHERE err_sqlstate = '22012' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + is($names, '[]', "failed query reports no relations (got: $names)"); + + my $truncated = psch_query_clickhouse( + "SELECT relations_truncated FROM pg_stat_ch.events_raw " . + "WHERE err_sqlstate = '22012' LIMIT 1" + ); + is($truncated, 'false', "empty relations on error is not flagged as truncated (got: $truncated)"); +}; + # Encoding correctness: special characters, every relkind, the truncation # marker, and the exact byte-budget boundary. @@ -405,13 +431,13 @@ unlike($names, qr/'sp_leaf_\d'/, "leaves two levels down not reported (got: $names)"); }; -subtest 'subpartitioned hierarchy past the old 1024-relation seen-set cap collapses correctly' => sub { +subtest 'subpartitioned hierarchy with a large range table collapses correctly' => sub { psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); psch_reset_stats($node); - # 600 (mid, leaf) pairs plus the root exceed PSCH_MAX_RELATIONS_LEN (1024) - # distinct relations in one range table -- regression test for the - # fixed-size Oid[1024] `seen` array this used to overflow into. + # 600 (mid, leaf) pairs plus the root: a large range table, to exercise + # the seen-set (a hash sized to the range table) well beyond a handful + # of entries. $node->safe_psql('postgres', 'CREATE TABLE big_root(id int, val text) PARTITION BY RANGE (id)'); $node->safe_psql('postgres', <<'SQL'); DO $do$ @@ -796,123 +822,6 @@ END unlike($names, qr/'old_name_tab'/, "no stale old name lingers anywhere (got: $names)"); }; -subtest 'relations nesting depth recovers after a run of unsampled queries' => sub { - psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); - psch_reset_stats($node); - - $node->safe_psql('postgres', 'CREATE TABLE nest_recover_t(a int)'); - - # 20 deterministically-unsampled queries (sample_rate=0, high - # min_duration_us) used to permanently saturate the nesting-depth stack. - my $sql = 'SET pg_stat_ch.sample_rate = 0.0; ' . - 'SET pg_stat_ch.min_duration_us = 2000000000; ' . - ('SELECT * FROM nest_recover_t; ' x 20) . - 'RESET pg_stat_ch.sample_rate; ' . - 'RESET pg_stat_ch.min_duration_us; ' . - 'SELECT * FROM nest_recover_t;'; - $node->safe_psql('postgres', $sql); - $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); - - my $names = psch_wait_for_clickhouse_query( - "SELECT relation_names FROM pg_stat_ch.events_raw " . - "WHERE query_text LIKE 'SELECT * FROM nest_recover_t%' ORDER BY ts DESC LIMIT 1", - sub { $_[0] ne '' }, - 10 - ); - like($names, qr/'nest_recover_t'/, - "relations still captured after a 20-query unsampled streak (got: $names)"); -}; - -subtest 'nesting depth recovers after permission-check failures inside a PL/pgSQL exception' => sub { - psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); - psch_reset_stats($node); - - $node->safe_psql('postgres', 'CREATE TABLE perm_fail_t(a int)'); - $node->safe_psql('postgres', 'CREATE ROLE perm_fail_role'); - - # Each iteration fails inside ExecutorStart itself (permission denied, - # before any execution), caught by the PL/pgSQL exception block -- which - # runs it as its own subtransaction. 20 of these used to leak a nesting - # level every time with no way to recover. - my $sql = - 'SET ROLE perm_fail_role; ' . - q{DO $$ BEGIN - FOR i IN 1..20 LOOP - BEGIN - EXECUTE 'SELECT * FROM perm_fail_t'; - EXCEPTION WHEN insufficient_privilege THEN NULL; - END; - END LOOP; - END $$;} . - ' RESET ROLE; ' . - 'SELECT * FROM perm_fail_t;'; - $node->safe_psql('postgres', $sql); - $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); - - my $names = psch_wait_for_clickhouse_query( - "SELECT relation_names FROM pg_stat_ch.events_raw " . - "WHERE query_text LIKE 'SELECT * FROM perm_fail_t%' ORDER BY ts DESC LIMIT 1", - sub { $_[0] ne '' }, - 10 - ); - like($names, qr/'perm_fail_t'/, - "relations still captured after 20 permission-check failures (got: $names)"); -}; - -subtest 'nesting depth recovers after a run of COPY statements' => sub { - psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); - psch_reset_stats($node); - - $node->safe_psql('postgres', 'CREATE TABLE copy_leak_t(a int)'); - - # COPY's own permission check bypasses ExecutorStart/End entirely, so 20 - # of these used to leak a nesting level every time with no way to recover. - my $sql = ("COPY copy_leak_t TO '/dev/null'; " x 20) . - 'SELECT * FROM copy_leak_t;'; - $node->safe_psql('postgres', $sql); - $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); - - my $names = psch_wait_for_clickhouse_query( - "SELECT relation_names FROM pg_stat_ch.events_raw " . - "WHERE query_text LIKE 'SELECT * FROM copy_leak_t%' ORDER BY ts DESC LIMIT 1", - sub { $_[0] ne '' }, - 10 - ); - like($names, qr/'copy_leak_t'/, - "relations still captured after 20 COPY statements (got: $names)"); -}; - -subtest 'nesting depth recovers after many distinct failed-cursor fetches' => sub { - psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); - psch_reset_stats($node); - - $node->safe_psql('postgres', 'CREATE TABLE cursor_leak_t(a int)'); - - # 20 distinct failed-cursor/savepoint sequences used to exhaust all 16 - # nesting slots; must run in one session (psql, on_error_stop => 0). - my $sql = "BEGIN;\n"; - for my $i (1 .. 20) { - $sql .= "DECLARE cursor_leak_cur_$i NO SCROLL CURSOR FOR SELECT * FROM cursor_leak_t;\n"; - $sql .= "SAVEPOINT cursor_leak_sp_$i;\n"; - $sql .= "FETCH BACKWARD FROM cursor_leak_cur_$i;\n"; - $sql .= "ROLLBACK TO SAVEPOINT cursor_leak_sp_$i;\n"; - $sql .= "RELEASE SAVEPOINT cursor_leak_sp_$i;\n"; - } - $sql .= "COMMIT;\n"; - $sql .= "SELECT * FROM cursor_leak_t;\n"; - $sql .= "SELECT pg_stat_ch_flush();\n"; - $node->psql('postgres', $sql, on_error_stop => 0); - - my $names = psch_wait_for_clickhouse_query( - "SELECT relation_names FROM pg_stat_ch.events_raw " . - "WHERE query_text LIKE 'SELECT * FROM cursor_leak_t%' ORDER BY ts DESC LIMIT 1", - sub { $_[0] ne '' }, - 10 - ); - like($names, qr/'cursor_leak_t'/, - "relations still captured after 20 distinct failed-cursor fetches (got: $names)"); -}; - subtest 'relation_* columns are omitted when disabled, so an unmigrated schema still accepts inserts' => sub { # Simulate a destination that hasn't run the relation_* columns # migration yet (the Down side of 20261104000003_*): the exporter must diff --git a/test/unit/otel_string_array_attr_test.cc b/test/unit/otel_string_array_attr_test.cc deleted file mode 100644 index 77b04a9..0000000 --- a/test/unit/otel_string_array_attr_test.cc +++ /dev/null @@ -1,61 +0,0 @@ -#include "export/otel_string_array_attr.h" - -#include - -using opentelemetry::proto::common::v1::KeyValue; - -TEST(SetStringArrayAttr, KeyIsSet) { - KeyValue kv; - psch::SetStringArrayAttr(&kv, "relation_names", {}); - EXPECT_EQ(kv.key(), "relation_names"); -} - -TEST(SetStringArrayAttr, EmptyInputYieldsEmptyArrayNotAbsentAttr) { - KeyValue kv; - psch::SetStringArrayAttr(&kv, "relation_names", {}); - ASSERT_TRUE(kv.value().has_array_value()); - EXPECT_EQ(kv.value().array_value().values_size(), 0); -} - -TEST(SetStringArrayAttr, SingleEntry) { - KeyValue kv; - psch::SetStringArrayAttr(&kv, "relation_names", {"public.foo1"}); - ASSERT_EQ(kv.value().array_value().values_size(), 1); - EXPECT_EQ(kv.value().array_value().values(0).string_value(), "public.foo1"); -} - -TEST(SetStringArrayAttr, MultipleEntriesPreserveOrder) { - KeyValue kv; - psch::SetStringArrayAttr(&kv, "relation_names", {"public.foo1", "public.foo2", "sch.foo3"}); - const auto& arr = kv.value().array_value(); - ASSERT_EQ(arr.values_size(), 3); - EXPECT_EQ(arr.values(0).string_value(), "public.foo1"); - EXPECT_EQ(arr.values(1).string_value(), "public.foo2"); - EXPECT_EQ(arr.values(2).string_value(), "sch.foo3"); -} - -TEST(SetStringArrayAttr, SingleCharElementSurvives) { - KeyValue kv; - psch::SetStringArrayAttr(&kv, "relation_kinds", {"r"}); - EXPECT_EQ(kv.value().array_value().values(0).string_value(), "r"); -} - -TEST(EstimateStringArrayAttrBytes, GrowsWithElementCountAndLength) { - size_t empty = psch::EstimateStringArrayAttrBytes("relation_names", {}); - size_t one_short = psch::EstimateStringArrayAttrBytes("relation_names", {"a.b"}); - size_t one_long = - psch::EstimateStringArrayAttrBytes("relation_names", {"public.a_much_longer_name"}); - size_t two = psch::EstimateStringArrayAttrBytes("relation_names", {"a.b", "c.d"}); - - EXPECT_GT(one_short, empty); - EXPECT_GT(one_long, one_short); - EXPECT_GT(two, one_short); -} - -TEST(EstimateStringArrayAttrBytes, CountsPerElementFramingOverhead) { - size_t zero = psch::EstimateStringArrayAttrBytes("k", {}); - size_t one = psch::EstimateStringArrayAttrBytes("k", {"abc"}); - // Each array element is its own nested AnyValue -- adding one should cost - // more than just its own string bytes. - EXPECT_GT(one - zero, 3u); -} diff --git a/test/unit/relations_decode_test.cc b/test/unit/relations_decode_test.cc deleted file mode 100644 index 52c0a82..0000000 --- a/test/unit/relations_decode_test.cc +++ /dev/null @@ -1,113 +0,0 @@ -#include "export/relations_decode.h" - -#include - -namespace { - -// Helper: build a blob byte-string from raw char values so tests read close -// to the wire format rather than fighting C string escaping. -std::string Blob(std::initializer_list bytes) { - return std::string(bytes.begin(), bytes.end()); -} - -} // namespace - -TEST(DecodeRelations, EmptyInputYieldsEmptyResult) { - auto out = DecodeRelations("", 0); - EXPECT_FALSE(out.truncated); - EXPECT_TRUE(out.namespaces.empty()); - EXPECT_TRUE(out.names.empty()); - EXPECT_TRUE(out.kinds.empty()); -} - -TEST(DecodeRelations, HeaderOnlyNoEntries) { - auto blob = Blob({0, 0}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - EXPECT_FALSE(out.truncated); - EXPECT_TRUE(out.namespaces.empty()); -} - -TEST(DecodeRelations, SingleEntry) { - // truncated=0, count=1, ns="public"(6), name="foo1"(4), kind='r' - auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 4, 'f', 'o', 'o', '1', 'r'}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - EXPECT_FALSE(out.truncated); - ASSERT_EQ(out.namespaces.size(), 1u); - EXPECT_EQ(out.namespaces[0], "public"); - EXPECT_EQ(out.names[0], "foo1"); - EXPECT_EQ(out.kinds[0], "r"); -} - -TEST(DecodeRelations, DotInsideNameSurvivesIntact) { - // A name containing a literal '.' must not be split -- this is the whole - // point of moving off the comma/dot-joined text format. - auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 10, 'w', 'e', 'i', 'r', - 'd', '.', 'n', 'a', 'm', 'e', 'r'}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - ASSERT_EQ(out.names.size(), 1u); - EXPECT_EQ(out.names[0], "weird.name"); -} - -TEST(DecodeRelations, CommaInsideNameSurvivesIntact) { - auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 9, 'h', 'a', 's', ',', - 'c', 'o', 'm', 'm', 'a', 'r'}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - ASSERT_EQ(out.names.size(), 1u); - EXPECT_EQ(out.names[0], "has,comma"); -} - -TEST(DecodeRelations, MultipleEntriesAndAllRelkinds) { - auto blob = Blob({ - 0, 3, - 6, 'p', 'u', 'b', 'l', 'i', 'c', 2, 'r', '1', 'r', - 6, 'p', 'u', 'b', 'l', 'i', 'c', 2, 'v', '1', 'v', - 6, 'p', 'u', 'b', 'l', 'i', 'c', 2, 'm', '1', 'm', - }); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - ASSERT_EQ(out.names.size(), 3u); - EXPECT_EQ(out.names[0], "r1"); - EXPECT_EQ(out.kinds[0], "r"); - EXPECT_EQ(out.names[1], "v1"); - EXPECT_EQ(out.kinds[1], "v"); - EXPECT_EQ(out.names[2], "m1"); - EXPECT_EQ(out.kinds[2], "m"); -} - -TEST(DecodeRelations, TruncatedFlagIsReported) { - auto blob = Blob({1, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 4, 'f', 'o', 'o', '1', 'r'}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - EXPECT_TRUE(out.truncated); -} - -TEST(DecodeRelations, TruncatedWithZeroCountYieldsEmptyButTruncated) { - // The blob CollectRelations emits when the budget is too small to fit even - // one entry -- must read as truncated, not as "zero relations". - auto blob = Blob({1, 0}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - EXPECT_TRUE(out.truncated); - EXPECT_TRUE(out.namespaces.empty()); -} - -TEST(DecodeRelations, LenShorterThanBufferOnlyReadsLenBytes) { - auto blob = Blob({0, 1, 6, 'p', 'u', 'b', 'l', 'i', 'c', 4, 'f', 'o', 'o', '1', 'r', - 'I', 'G', 'N', 'O', 'R', 'E', 'D'}); - auto out = DecodeRelations(blob.data(), 15); // 15 == real entry length - ASSERT_EQ(out.names.size(), 1u); - EXPECT_EQ(out.names[0], "foo1"); -} - -TEST(DecodeRelations, MalformedLengthDoesNotReadOutOfBounds) { - // Declares count=1, but ns_len claims 200 bytes when only 3 remain. - // Must stop cleanly, not read/crash past the buffer. - auto blob = Blob({0, 1, 200, 'a', 'b', 'c'}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - EXPECT_TRUE(out.names.empty()); -} - -TEST(DecodeRelations, OomSentinelReportsTruncatedNoEntries) { - // len==1 is the DSA-OOM sentinel (see relations_budget.h). - auto blob = Blob({0}); - auto out = DecodeRelations(blob.data(), static_cast(blob.size())); - EXPECT_TRUE(out.truncated); - EXPECT_TRUE(out.namespaces.empty()); -} diff --git a/vcpkg.json b/vcpkg.json index bff3552..d2720ee 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -14,7 +14,6 @@ }, "lz4", "openssl", - "zstd", - "gtest" + "zstd" ] } From 286746ff4f381eb24323502e516f4e45cc9e40ba Mon Sep 17 00:00:00 2001 From: Aliaksei Andryiashau Date: Tue, 17 Nov 2026 18:08:05 +0100 Subject: [PATCH 4/5] fix: clean up GTest-removal leftovers, shrink seen-set, add inheritance TAP case - Drop the now-unused -DPSCH_BUILD_UNIT_TESTS=OFF Dockerfile arg and restore CMakeLists.txt's trailing blank line, so neither appears in the diff. - Drop stale header comments pointing at deleted test/unit/*_test.cc files. - Size CollectRelations' seen-set with a small fixed constant instead of scaling it to the range table: the byte budget caps it at ~200 entries regardless, and planner-generated children never reach it. - Add a TAP subtest for plain-inheritance collapse (t/039), mirroring the existing partition-collapse coverage. Addresses jsc0218's 2026-11-17 review comments. --- CMakeLists.txt | 1 + docker/postgres-ext.Dockerfile | 1 - src/export/otel_string_array_attr.h | 2 -- src/export/relations_decode.h | 3 +-- src/hooks/hooks.c | 12 +++++++----- t/039_relations_export.pl | 19 +++++++++++++++++++ 6 files changed, 28 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ea3957..157f7c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,3 +145,4 @@ install(TARGETS pg_stat_ch LIBRARY DESTINATION ${PostgreSQLServer_PKGLIB_DIR}) install(FILES pg_stat_ch.control DESTINATION ${PostgreSQLServer_SHARE_DIR}/extension) file(GLOB SQL_FILES sql/pg_stat_ch--*.sql) install(FILES ${SQL_FILES} DESTINATION ${PostgreSQLServer_SHARE_DIR}/extension) + diff --git a/docker/postgres-ext.Dockerfile b/docker/postgres-ext.Dockerfile index a1e2cda..ada94f9 100644 --- a/docker/postgres-ext.Dockerfile +++ b/docker/postgres-ext.Dockerfile @@ -61,7 +61,6 @@ COPY third_party/clickhouse-c/ third_party/clickhouse-c/ RUN cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DPSCH_BUILD_UNIT_TESTS=OFF \ -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ -DVCPKG_TARGET_TRIPLET="$(cat /build/vcpkg-triplet)" \ -DVCPKG_HOST_TRIPLET="$(cat /build/vcpkg-triplet)" \ diff --git a/src/export/otel_string_array_attr.h b/src/export/otel_string_array_attr.h index 0c572d5..4508e59 100644 --- a/src/export/otel_string_array_attr.h +++ b/src/export/otel_string_array_attr.h @@ -1,6 +1,4 @@ // Builds an array-valued OTLP attribute for any HC string-array stat. -// Separate from otel_exporter.cc so it's unit-testable without a live -// gRPC channel (test/unit/otel_string_array_attr_test.cc). #ifndef PG_STAT_CH_SRC_EXPORT_OTEL_STRING_ARRAY_ATTR_H_ #define PG_STAT_CH_SRC_EXPORT_OTEL_STRING_ARRAY_ATTR_H_ diff --git a/src/export/relations_decode.h b/src/export/relations_decode.h index b79d1df..8b7a7a9 100644 --- a/src/export/relations_decode.h +++ b/src/export/relations_decode.h @@ -1,5 +1,4 @@ -// Pure C++, no PostgreSQL headers -- kept separate from stats_exporter.cc so -// it's unit-testable in isolation (see test/unit/relations_decode_test.cc). +// Pure C++, no PostgreSQL headers: decodes the relations blob built by CollectRelations in hooks.c. #ifndef PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ #define PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ diff --git a/src/hooks/hooks.c b/src/hooks/hooks.c index 7efdde3..0e58a18 100644 --- a/src/hooks/hooks.c +++ b/src/hooks/hooks.c @@ -869,10 +869,10 @@ bool PschSuppressErrorCapture(bool suppress) { return prev; } -// Oid set for CollectRelations' dedup/collapse below. Created fresh per call -// in the caller's es_query_cxt -- sized to the range table itself, not the -// relations_buf byte budget, since a collapsed partition consumes a slot -// here without consuming any budget bytes. +// Oid set for CollectRelations' dedup below. Created fresh per call in the +// caller's es_query_cxt. Planner-generated partition/inheritance children are +// skipped before they reach this set, so it stays small: it is bounded by the +// byte budget, not by the range-table length. typedef struct PschRelidSeenEntry { Oid relid; char status; @@ -893,7 +893,9 @@ typedef struct PschRelidSeenEntry { // [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count. static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos) { int nrt = list_length(rt); - psch_relid_seen_hash* seen = psch_relid_seen_create(CurrentMemoryContext, Max(nrt, 1), NULL); + // Starting size only: simplehash grows on demand, and the byte budget caps + // this set at roughly 200 relations, so don't scale it with the range table. + psch_relid_seen_hash* seen = psch_relid_seen_create(CurrentMemoryContext, 64, NULL); uint8 count = 0; bool truncated = false; int rti = 0; diff --git a/t/039_relations_export.pl b/t/039_relations_export.pl index 060c447..381a76e 100644 --- a/t/039_relations_export.pl +++ b/t/039_relations_export.pl @@ -494,6 +494,25 @@ END like($names, qr/'gp_part_0'/, "directly-queried partition still reported (got: $names)"); }; +subtest 'inheritance children collapse into the parent' => sub { + psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); + psch_reset_stats($node); + + $node->safe_psql('postgres', 'CREATE TABLE inh_parent(a int)'); + $node->safe_psql('postgres', 'CREATE TABLE inh_child1() INHERITS (inh_parent)'); + $node->safe_psql('postgres', 'CREATE TABLE inh_child2() INHERITS (inh_parent)'); + $node->safe_psql('postgres', 'SELECT * FROM inh_parent'); + $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()'); + + my $names = psch_wait_for_clickhouse_query( + "SELECT relation_names FROM pg_stat_ch.events_raw WHERE query_text LIKE 'SELECT * FROM inh_parent%' LIMIT 1", + sub { $_[0] ne '' }, + 10 + ); + like($names, qr/'inh_parent'/, "inheritance parent reported (got: $names)"); + unlike($names, qr/'inh_child/, "inheritance children collapsed into the parent (got: $names)"); +}; + subtest 'partition named explicitly alongside its parent is reported regardless of FROM order' => sub { psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw"); psch_reset_stats($node); From 6fa0cfa6ac144ebcbdf4431d1accb9ef8ce269ca Mon Sep 17 00:00:00 2001 From: Aliaksei Andryiashau Date: Tue, 17 Nov 2026 19:12:33 +0100 Subject: [PATCH 5/5] refactor: extract relations collection into its own file and split it - Move PschRelidSeenEntry, the simplehash instantiation, and CollectRelations out of hooks.c into relations_collect.c/.h, dropping the forward declaration (item 7). - Split CollectRelations into BuildChildMap, ShouldReportRte, and AppendRelationEntry (item 8). Addresses jsc0218's 2026-11-17 review comments. --- src/export/relations_decode.h | 5 +- src/hooks/hooks.c | 116 +------------------------- src/hooks/relations_collect.c | 148 ++++++++++++++++++++++++++++++++++ src/hooks/relations_collect.h | 23 ++++++ src/queue/event.h | 5 +- 5 files changed, 178 insertions(+), 119 deletions(-) create mode 100644 src/hooks/relations_collect.c create mode 100644 src/hooks/relations_collect.h diff --git a/src/export/relations_decode.h b/src/export/relations_decode.h index 8b7a7a9..fbdf887 100644 --- a/src/export/relations_decode.h +++ b/src/export/relations_decode.h @@ -1,4 +1,5 @@ -// Pure C++, no PostgreSQL headers: decodes the relations blob built by CollectRelations in hooks.c. +// Pure C++, no PostgreSQL headers: decodes the relations blob built by +// CollectRelations in relations_collect.c. #ifndef PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ #define PG_STAT_CH_SRC_EXPORT_RELATIONS_DECODE_H_ @@ -8,7 +9,7 @@ #include "hooks/relations_budget.h" -// Decoded form of the relations blob built by CollectRelations in hooks.c: +// Decoded form of the relations blob built by CollectRelations in relations_collect.c: // [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count struct DecodedRelations { bool truncated = false; diff --git a/src/hooks/hooks.c b/src/hooks/hooks.c index 0e58a18..48f8d60 100644 --- a/src/hooks/hooks.c +++ b/src/hooks/hooks.c @@ -6,9 +6,7 @@ #include "access/parallel.h" #include "access/xact.h" -#include "catalog/pg_class.h" #include "commands/dbcommands.h" -#include "common/hashfn.h" #include "common/ip.h" #include "common/pg_prng.h" #include "executor/executor.h" @@ -17,16 +15,13 @@ #include "lib/stringinfo.h" #include "libpq/libpq-be.h" #include "miscadmin.h" -#include "nodes/pathnodes.h" #include "nodes/queryjumble.h" #include "parser/analyze.h" -#include "parser/parsetree.h" #include "postmaster/bgworker.h" #include "storage/proc.h" #include "tcop/utility.h" #include "utils/elog.h" #include "utils/guc.h" -#include "utils/lsyscache.h" #include "utils/timestamp.h" #include "compat.h" @@ -34,7 +29,7 @@ #include "hooks/hooks.h" #include "hooks/query_normalize.h" #include "hooks/query_normalize_state.h" -#include "hooks/relations_budget.h" +#include "hooks/relations_collect.h" #include "hooks/string_utils.h" #include "queue/event.h" #include "queue/shmem.h" @@ -310,7 +305,6 @@ static void CopyQueryText(PschEvent* event, uint64 query_id) { sizeof(event->query), &event->query_len); } -static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos); // Relations are collected only here, at ExecutorEnd, so only queries that run // to completion carry them. A query that errors out (including failures before // execution starts, e.g. permission denied) never reaches ExecutorEnd, and its @@ -869,114 +863,6 @@ bool PschSuppressErrorCapture(bool suppress) { return prev; } -// Oid set for CollectRelations' dedup below. Created fresh per call in the -// caller's es_query_cxt. Planner-generated partition/inheritance children are -// skipped before they reach this set, so it stays small: it is bounded by the -// byte budget, not by the range-table length. -typedef struct PschRelidSeenEntry { - Oid relid; - char status; -} PschRelidSeenEntry; - -#define SH_PREFIX psch_relid_seen -#define SH_ELEMENT_TYPE PschRelidSeenEntry -#define SH_KEY_TYPE Oid -#define SH_KEY relid -#define SH_HASH_KEY(tb, key) murmurhash32((uint32)(key)) -#define SH_EQUAL(tb, a, b) ((a) == (b)) -#define SH_SCOPE static inline -#define SH_DECLARE -#define SH_DEFINE -#include "lib/simplehash.h" - -// Builds the relations blob: -// [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count. -static void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos) { - int nrt = list_length(rt); - // Starting size only: simplehash grows on demand, and the byte budget caps - // this set at roughly 200 relations, so don't scale it with the range table. - psch_relid_seen_hash* seen = psch_relid_seen_create(CurrentMemoryContext, 64, NULL); - uint8 count = 0; - bool truncated = false; - int rti = 0; - ListCell* lc; - - // Defense in depth -- the GUC's own min bound (guc.c) already enforces this. - Assert(psch_relations_max_len >= PSCH_RELATIONS_HEADER_LEN); - - Index* parent_of_rti = palloc0(sizeof(Index) * (nrt + 1)); - ListCell* alc; - foreach (alc, append_rel_infos) { - AppendRelInfo* appinfo = (AppendRelInfo*)lfirst(alc); - Assert(appinfo->child_relid >= 1 && (int)appinfo->child_relid <= nrt); - parent_of_rti[appinfo->child_relid] = appinfo->parent_relid; - } - - // Reserve the header now; patched with real values once known. - appendStringInfoChar(buf, 0); - appendStringInfoChar(buf, 0); - - foreach (lc, rt) { - rti++; - RangeTblEntry* rte = (RangeTblEntry*)lfirst(lc); - - // Plain tables, plus view-expansion artifacts: the rewriter turns a - // referenced view into an RTE_SUBQUERY wrapping its definition, but - // stamps relkind='v' on that entry so it's still identifiable as a view. - // RTE_RELATION alone already covers every other relkind (matview, - // partitioned, foreign table) uniformly -- unaffected by that work. - // Partition and inheritance children are collapsed into their parent - // below: the parent is reported, planner-generated children are not. - if (rte->rtekind != RTE_RELATION && rte->relkind != RELKIND_VIEW) { - continue; - } - - // Collapse planner-generated children before deduplicating relation OIDs, - // so a later explicit reference to the same partition is still reported. - Index parent_rti = parent_of_rti[rti]; - if (parent_rti != 0 && rt_fetch(parent_rti, rt)->rtekind == RTE_RELATION) { - continue; - } - - bool found; - psch_relid_seen_insert(seen, rte->relid, &found); - if (found) { - continue; - } - - const char* namespace_name = get_namespace_name(get_rel_namespace(rte->relid)); - const char* relation_name = get_rel_name(rte->relid); - if (namespace_name == NULL || relation_name == NULL) { - continue; // Relation dropped concurrently; skip rather than crash. - } - - // NAMEDATALEN-1 (63) max per Postgres identifier rules, so these always - // fit in a uint8 length prefix. - size_t ns_len = strlen(namespace_name); - size_t name_len = strlen(relation_name); - Assert(ns_len < NAMEDATALEN); - Assert(name_len < NAMEDATALEN); - - // No separator byte to account for -- every entry is self-delimiting. - int entry_len = (int)(1 + ns_len + 1 + name_len + 1); - if (buf->len + entry_len > psch_relations_max_len) { - truncated = true; - break; // Budget exhausted -- truncate, same as query text does. - } - - appendStringInfoChar(buf, (char)ns_len); - appendBinaryStringInfo(buf, namespace_name, (int)ns_len); - appendStringInfoChar(buf, (char)name_len); - appendBinaryStringInfo(buf, relation_name, (int)name_len); - appendStringInfoChar(buf, (char)rte->relkind); - count++; - } - - buf->data[0] = truncated ? 1 : 0; - buf->data[1] = (char)count; - psch_relid_seen_destroy(seen); -} - void PschInstallHooks(void) { EnableQueryId(); diff --git a/src/hooks/relations_collect.c b/src/hooks/relations_collect.c new file mode 100644 index 0000000..f02cda8 --- /dev/null +++ b/src/hooks/relations_collect.c @@ -0,0 +1,148 @@ +#include "postgres.h" + +#include "catalog/pg_class.h" +#include "common/hashfn.h" +#include "nodes/pathnodes.h" +#include "parser/parsetree.h" +#include "utils/lsyscache.h" + +#include "config/guc.h" +#include "hooks/relations_budget.h" +#include "hooks/relations_collect.h" + +// Oid set for CollectRelations' dedup below. Created fresh per call in the +// caller's es_query_cxt. Planner-generated partition/inheritance children are +// skipped before they reach this set, so it stays small: it is bounded by the +// byte budget, not by the range-table length. +typedef struct PschRelidSeenEntry { + Oid relid; + char status; +} PschRelidSeenEntry; + +#define SH_PREFIX psch_relid_seen +#define SH_ELEMENT_TYPE PschRelidSeenEntry +#define SH_KEY_TYPE Oid +#define SH_KEY relid +#define SH_HASH_KEY(tb, key) murmurhash32((uint32)(key)) +#define SH_EQUAL(tb, a, b) ((a) == (b)) +#define SH_SCOPE static inline +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +// Fills parent_of_rti[child's rti] = parent's rti for every planner-generated +// partition/inheritance child; everything else stays 0 (the palloc0 default). +// Indexed by range-table index (1..nrt), so parent_of_rti must have nrt+1 +// elements. +static void BuildChildMap(Index* parent_of_rti, int nrt, List* append_rel_infos) { + (void)nrt; // Only used by the Assert below, which compiles out in release builds. + ListCell* alc; + foreach (alc, append_rel_infos) { + AppendRelInfo* appinfo = (AppendRelInfo*)lfirst(alc); + Assert(appinfo->child_relid >= 1 && (int)appinfo->child_relid <= nrt); + parent_of_rti[appinfo->child_relid] = appinfo->parent_relid; + } +} + +// True if the range-table entry at rti should be considered for reporting. +// +// Plain tables, plus view-expansion artifacts: the rewriter turns a +// referenced view into an RTE_SUBQUERY wrapping its definition, but stamps +// relkind='v' on that entry so it's still identifiable as a view. +// RTE_RELATION alone already covers every other relkind (matview, +// partitioned, foreign table) uniformly -- unaffected by that work. +// +// Partition and inheritance children are collapsed into their parent: the +// parent is reported, planner-generated children are not. Collapsing happens +// before deduplication, so a later explicit reference to the same partition +// is still reported. +static bool ShouldReportRte(RangeTblEntry* rte, List* rt, Index* parent_of_rti, int rti) { + if (rte->rtekind != RTE_RELATION && rte->relkind != RELKIND_VIEW) { + return false; + } + + Index parent_rti = parent_of_rti[rti]; + if (parent_rti != 0 && rt_fetch(parent_rti, rt)->rtekind == RTE_RELATION) { + return false; + } + + return true; +} + +// Appends one [ns_len][ns][name_len][name][relkind] entry to buf. Returns +// false (buf unchanged) if the entry would exceed psch_relations_max_len, +// signaling the caller to stop and flag truncation, same as query text does. +static bool AppendRelationEntry(StringInfo buf, const char* namespace_name, + const char* relation_name, char relkind) { + // NAMEDATALEN-1 (63) max per Postgres identifier rules, so these always + // fit in a uint8 length prefix. + size_t ns_len = strlen(namespace_name); + size_t name_len = strlen(relation_name); + Assert(ns_len < NAMEDATALEN); + Assert(name_len < NAMEDATALEN); + + // No separator byte to account for -- every entry is self-delimiting. + int entry_len = (int)(1 + ns_len + 1 + name_len + 1); + if (buf->len + entry_len > psch_relations_max_len) { + return false; + } + + appendStringInfoChar(buf, (char)ns_len); + appendBinaryStringInfo(buf, namespace_name, (int)ns_len); + appendStringInfoChar(buf, (char)name_len); + appendBinaryStringInfo(buf, relation_name, (int)name_len); + appendStringInfoChar(buf, (char)relkind); + return true; +} + +void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos) { + int nrt = list_length(rt); + // Starting size only: simplehash grows on demand, and the byte budget caps + // this set at roughly 200 relations, so don't scale it with the range table. + psch_relid_seen_hash* seen = psch_relid_seen_create(CurrentMemoryContext, 64, NULL); + uint8 count = 0; + bool truncated = false; + int rti = 0; + ListCell* lc; + + // Defense in depth -- the GUC's own min bound (guc.c) already enforces this. + Assert(psch_relations_max_len >= PSCH_RELATIONS_HEADER_LEN); + + Index* parent_of_rti = palloc0(sizeof(Index) * (nrt + 1)); + BuildChildMap(parent_of_rti, nrt, append_rel_infos); + + // Reserve the header now; patched with real values once known. + appendStringInfoChar(buf, 0); + appendStringInfoChar(buf, 0); + + foreach (lc, rt) { + rti++; + RangeTblEntry* rte = (RangeTblEntry*)lfirst(lc); + + if (!ShouldReportRte(rte, rt, parent_of_rti, rti)) { + continue; + } + + bool found; + psch_relid_seen_insert(seen, rte->relid, &found); + if (found) { + continue; + } + + const char* namespace_name = get_namespace_name(get_rel_namespace(rte->relid)); + const char* relation_name = get_rel_name(rte->relid); + if (namespace_name == NULL || relation_name == NULL) { + continue; // Relation dropped concurrently; skip rather than crash. + } + + if (!AppendRelationEntry(buf, namespace_name, relation_name, (char)rte->relkind)) { + truncated = true; + break; + } + count++; + } + + buf->data[0] = truncated ? 1 : 0; + buf->data[1] = (char)count; + psch_relid_seen_destroy(seen); +} diff --git a/src/hooks/relations_collect.h b/src/hooks/relations_collect.h new file mode 100644 index 0000000..498db7b --- /dev/null +++ b/src/hooks/relations_collect.h @@ -0,0 +1,23 @@ +// Builds the relations blob reported per query: the tables/views/matviews +// the executor touched, deduplicated, with partition and inheritance +// children collapsed into their parent. Layout: +// [truncated:u8][count:u8]{[ns_len:u8][ns][name_len:u8][name][relkind:u8]}*count. +#ifndef PG_STAT_CH_SRC_HOOKS_RELATIONS_COLLECT_H_ +#define PG_STAT_CH_SRC_HOOKS_RELATIONS_COLLECT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "postgres.h" + +#include "lib/stringinfo.h" +#include "nodes/pg_list.h" + +void CollectRelations(StringInfo buf, List* rt, List* append_rel_infos); + +#ifdef __cplusplus +} +#endif + +#endif // PG_STAT_CH_SRC_HOOKS_RELATIONS_COLLECT_H_ diff --git a/src/queue/event.h b/src/queue/event.h index af7f727..c527095 100644 --- a/src/queue/event.h +++ b/src/queue/event.h @@ -43,8 +43,9 @@ extern "C" { // Maximum error message length (truncated if longer) #define PSCH_MAX_ERR_MSG_LEN 2048 -// Maximum length of the relations blob (layout in hooks.c, CollectRelations), -// truncated on overflow like query text -- truncation is self-flagged. +// Maximum length of the relations blob (layout in relations_collect.c, +// CollectRelations), truncated on overflow like query text -- truncation is +// self-flagged. #define PSCH_MAX_RELATIONS_LEN 1024 // Maximum application name length