Repository navigation
feat: collect and export a relations array per query #135
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5b19437
8734538
2e71af5
286746f
6fa0cfa
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| #ifndef PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ | ||
| #define PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ | ||
|
|
||
| // [truncated:u8][count:u8] -- the GUC's minimum must never go below this. | ||
| #define PSCH_RELATIONS_HEADER_LEN 2 | ||
|
|
||
| // Sentinel relations_len for "blob lost to DSA OOM" -- distinct from 0 (no | ||
| // relations) and real blob lengths (>= PSCH_RELATIONS_HEADER_LEN). | ||
| #define PSCH_RELATIONS_OOM_LEN 1 | ||
|
|
||
| #endif // PG_STAT_CH_INCLUDE_HOOKS_RELATIONS_BUDGET_H_ |
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,19 @@ | ||||||
| -- Adds structured relation columns: namespace, name, and relkind kept as | ||||||
| -- separate parallel arrays. Collected in src/hooks/hooks.c at ExecutorEnd. | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Stale pointer after the refactor:
Suggested change
|
||||||
|
|
||||||
| -- +goose Up | ||||||
|
|
||||||
| -- +goose StatementBegin | ||||||
| ALTER TABLE pg_stat_ch.events_raw | ||||||
| ADD COLUMN IF NOT EXISTS relation_namespaces Array(LowCardinality(String)) COMMENT 'Schema name per entry, positionally parallel to relation_names/relation_kinds.', | ||||||
| ADD COLUMN IF NOT EXISTS relation_names Array(LowCardinality(String)) COMMENT 'Relation name per entry, positionally parallel to relation_namespaces/relation_kinds.', | ||||||
| ADD COLUMN IF NOT EXISTS relation_kinds Array(FixedString(1)) COMMENT 'pg_class.relkind char per entry (r=table, v=view, m=matview, f=foreign table, p=partitioned table, ...).', | ||||||
| ADD COLUMN IF NOT EXISTS relations_truncated Bool DEFAULT false COMMENT 'True if pg_stat_ch.relations_max_len was exceeded and this list is incomplete.'; | ||||||
| -- +goose StatementEnd | ||||||
|
|
||||||
| -- +goose Down | ||||||
|
|
||||||
| -- Keep the columns on rollback: ClickHouse refuses to drop them while | ||||||
| -- events_recent_1h's SELECT * depends on them, and producers that don't send | ||||||
| -- them still insert. | ||||||
| SELECT 1; | ||||||
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -7,6 +7,7 @@ | |||||||||
| #include <limits.h> | ||||||||||
|
|
||||||||||
| #include "config/guc.h" | ||||||||||
| #include "hooks/relations_budget.h" | ||||||||||
|
|
||||||||||
| // GUC variable storage | ||||||||||
| bool psch_enabled = true; | ||||||||||
|
|
@@ -40,6 +41,8 @@ bool psch_use_unified_arrow_exporter = false; | |||||||||
| int psch_otel_max_block_bytes = 3 * 1024 * 1024; // 3 MiB (max: 16 MiB) | ||||||||||
| char* psch_extra_attributes = NULL; | ||||||||||
| char* psch_debug_arrow_dump_dir = NULL; | ||||||||||
| int psch_relations_max_len = 1023; // bytes, leaves room for the NUL in PSCH_MAX_RELATIONS_LEN=1024 | ||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The "room for the NUL" rationale doesn't apply to the relations blob The relations blob is length-prefixed binary ( Either raise the max to |
||||||||||
| bool psch_relations_enabled = false; | ||||||||||
|
|
||||||||||
| // Log level options (matches PostgreSQL's server_message_level_options pattern) | ||||||||||
| // clang-format off | ||||||||||
|
|
@@ -410,6 +413,35 @@ void PschInitGuc(void) { | |||||||||
| 0, | ||||||||||
| NULL, NULL, NULL); | ||||||||||
|
|
||||||||||
| DefineCustomIntVariable( | ||||||||||
| "pg_stat_ch.relations_max_len", | ||||||||||
| "Maximum bytes for the encoded list of tables/views a query touches.", | ||||||||||
| "Mirrors the query-text truncation pattern: the relations blob is a fixed " | ||||||||||
| "backend-local buffer, truncated (not grown) when a query touches enough " | ||||||||||
|
Comment on lines
+419
to
+420
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This long description is out of date after the move to ExecutorEnd: the relations blob is no longer a fixed backend-local buffer.
Suggested change
|
||||||||||
| "distinct relations to exceed this budget.", | ||||||||||
| &psch_relations_max_len, | ||||||||||
| 1023, // bootValue | ||||||||||
| PSCH_RELATIONS_HEADER_LEN, // min -- the header alone must still fit | ||||||||||
| 1023, // max (PSCH_MAX_RELATIONS_LEN - 1, room for the NUL) | ||||||||||
| PGC_SUSET, | ||||||||||
| 0, | ||||||||||
| NULL, NULL, NULL); | ||||||||||
|
|
||||||||||
| DefineCustomBoolVariable( | ||||||||||
| "pg_stat_ch.relations_enabled", | ||||||||||
| "Collect the relations array (tables/views a query touches).", | ||||||||||
| "When enabled, each backend resolves a query's touched relations from " | ||||||||||
| "its plan's range table at ExecutorEnd and exports them as the " | ||||||||||
| "relation_namespaces/relation_names/relation_kinds/relations_truncated " | ||||||||||
| "columns. Apply the relations schema migrations before enabling. " | ||||||||||
| "PGC_POSTMASTER: the ClickHouse exporter keeps its column set for the " | ||||||||||
| "life of the bgworker, so a runtime change would leave a stale column.", | ||||||||||
| &psch_relations_enabled, | ||||||||||
| false, | ||||||||||
| PGC_POSTMASTER, | ||||||||||
|
AliakseiAndryiashau2026 marked this conversation as resolved.
Resolved
|
||||||||||
| 0, | ||||||||||
| NULL, NULL, NULL); | ||||||||||
|
|
||||||||||
| DefineCustomBoolVariable( | ||||||||||
| "pg_stat_ch.track_labels", | ||||||||||
| "Extract sqlcommenter labels from query comments.", | ||||||||||
|
|
||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Small doc gap:
relations_truncatedis also set when the whole blob was lost because the DSA string area was out of memory (the decoder reportstruncated = truewith empty arrays for thePSCH_RELATIONS_OOM_LENsentinel). As written, a reader would assume the arrays only ever lose their tail after exceedingrelations_max_len. Suggested wording: