From 120f1a9ce58dcc24f0c83e1f2e0d686d5a02ec5b Mon Sep 17 00:00:00 2001 From: sdairs Date: Tue, 17 Nov 2026 17:40:50 +0000 Subject: [PATCH] fix(api): add useJsonNativeFormat to MySQL and Postgres ClickPipe settings Refreshes the snapshot to the live spec, which adds optional `useJsonNativeFormat` to `ClickPipeMySQLPipeSettings` and `ClickPipePostgresPipeSettings`. The field is create-only and organization-gated, so it is `Option` on all four types, with an optionality exemption on the Postgres request type, whose schema has no `required[]`. The CLI passes `None`, so its requests are unchanged. Co-Authored-By: Claude Opus 5.5 --- crates/clickhouse-cloud-api/CHANGELOG.md | 8 +++ .../clickhouse_cloud_openapi.json | 12 +++- .../src/models/clickpipes/mysql.rs | 13 +++++ .../src/models/clickpipes/postgres.rs | 13 +++++ .../tests/clickpipes/smoke_test.rs | 1 + .../tests/clickpipes/stages/mysql.rs | 1 + .../tests/clickpipes/stages/postgres.rs | 1 + .../tests/models/clickpipes.rs | 55 +++++++++++++++++++ .../clickhouse-openapi-analyzer/src/config.rs | 6 ++ .../src/cloud/clickpipes/sources/mysql.rs | 1 + .../src/cloud/clickpipes/sources/postgres.rs | 1 + 11 files changed, 111 insertions(+), 1 deletion(-) diff --git a/crates/clickhouse-cloud-api/CHANGELOG.md b/crates/clickhouse-cloud-api/CHANGELOG.md index 77b9a35d..0295e670 100644 --- a/crates/clickhouse-cloud-api/CHANGELOG.md +++ b/crates/clickhouse-cloud-api/CHANGELOG.md @@ -253,6 +253,14 @@ from the response type for writing a fetched list back. Get needs `control-plane:postgres-service:view`; the others need `control-plane:postgres-service:manage`. +`ClickPipeMySQLPipeSettings`, `ClickPipePostgresPipeSettings` and their +`*Response` types gain optional `use_json_native_format` +(`useJsonNativeFormat`). `Some(true)` stores JSON columns in native +ClickHouse JSON format (ClickHouse 25.3+) instead of `String`; for Postgres, +`json[]` and `jsonb[]` become `Array(JSON)`. It can be set only on create, and +only in organizations with the feature enabled, so `None` omits it. +Struct-literal callers need `use_json_native_format: None`. + ## 0.5.1 No changes to this crate; released in lockstep with `clickhousectl` 0.5.1. diff --git a/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json b/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json index 482e9af9..4e57c8f6 100644 --- a/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json +++ b/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json @@ -5,7 +5,7 @@ "version": "1.0", "contact": { "name": "ClickHouse Support", - "url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-1443100", + "url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-1443919", "email": "support@clickhouse.com" } }, @@ -30456,6 +30456,11 @@ "description": "Enable hard delete behavior in ReplacingMergeTree for PostgreSQL DELETE operations.", "type": "boolean", "example": false + }, + "useJsonNativeFormat": { + "description": "Store json and jsonb columns in native ClickHouse JSON format (requires ClickHouse 25.3+). Non-object values are wrapped as {\"_value\": ...}, and json[]/jsonb[] become Array(JSON). When disabled, JSON data is stored as String. Can only be set at creation. Must be enabled for your organization.", + "type": "boolean", + "example": false } } }, @@ -30931,6 +30936,11 @@ "description": "Enable hard delete behavior in ReplacingMergeTree for MySQL DELETE operations.", "type": "boolean", "example": false + }, + "useJsonNativeFormat": { + "description": "Store JSON columns in native ClickHouse JSON format (requires ClickHouse 25.3+). Non-object values are wrapped as {\"_value\": ...}. When disabled, JSON data is stored as String. Can only be set at creation. Must be enabled for your organization.", + "type": "boolean", + "example": false } }, "required": [ diff --git a/crates/clickhouse-cloud-api/src/models/clickpipes/mysql.rs b/crates/clickhouse-cloud-api/src/models/clickpipes/mysql.rs index 64f0417f..80b874b9 100644 --- a/crates/clickhouse-cloud-api/src/models/clickpipes/mysql.rs +++ b/crates/clickhouse-cloud-api/src/models/clickpipes/mysql.rs @@ -340,6 +340,14 @@ pub struct ClickPipeMySQLPipeSettings { pub sync_interval_seconds: Option, #[serde(rename = "useCompression", skip_serializing_if = "Option::is_none")] pub use_compression: Option, + /// Store JSON columns in native ClickHouse JSON format (ClickHouse 25.3+); + /// otherwise they are stored as `String`. Create only, and only for + /// organizations with the feature enabled. + #[serde( + rename = "useJsonNativeFormat", + skip_serializing_if = "Option::is_none" + )] + pub use_json_native_format: Option, } /// `ClickPipeMySQLPipeSettings` from the ClickHouse Cloud API, in response @@ -388,6 +396,11 @@ pub struct ClickPipeMySQLPipeSettingsResponse { pub sync_interval_seconds: Option, #[serde(rename = "useCompression", skip_serializing_if = "Option::is_none")] pub use_compression: Option, + #[serde( + rename = "useJsonNativeFormat", + skip_serializing_if = "Option::is_none" + )] + pub use_json_native_format: Option, } /// `ClickPipeMySQLPipeTableMapping` from the ClickHouse Cloud API. diff --git a/crates/clickhouse-cloud-api/src/models/clickpipes/postgres.rs b/crates/clickhouse-cloud-api/src/models/clickpipes/postgres.rs index 39845d37..b44d018b 100644 --- a/crates/clickhouse-cloud-api/src/models/clickpipes/postgres.rs +++ b/crates/clickhouse-cloud-api/src/models/clickpipes/postgres.rs @@ -369,6 +369,14 @@ pub struct ClickPipePostgresPipeSettings { skip_serializing_if = "Option::is_none" )] pub sync_interval_seconds: Option, + /// Store JSON columns in native ClickHouse JSON format (ClickHouse 25.3+); + /// otherwise they are stored as `String`. Create only, and only for + /// organizations with the feature enabled. + #[serde( + rename = "useJsonNativeFormat", + skip_serializing_if = "Option::is_none" + )] + pub use_json_native_format: Option, } /// `ClickPipePostgresPipeSettings` from the ClickHouse Cloud API, in response @@ -422,6 +430,11 @@ pub struct ClickPipePostgresPipeSettingsResponse { skip_serializing_if = "Option::is_none" )] pub sync_interval_seconds: Option, + #[serde( + rename = "useJsonNativeFormat", + skip_serializing_if = "Option::is_none" + )] + pub use_json_native_format: Option, } /// `ClickPipePostgresPipeTableMapping` from the ClickHouse Cloud API. diff --git a/crates/clickhouse-cloud-api/tests/clickpipes/smoke_test.rs b/crates/clickhouse-cloud-api/tests/clickpipes/smoke_test.rs index bf473629..5f392d43 100644 --- a/crates/clickhouse-cloud-api/tests/clickpipes/smoke_test.rs +++ b/crates/clickhouse-cloud-api/tests/clickpipes/smoke_test.rs @@ -481,6 +481,7 @@ async fn cloud_clickpipe_create_mysql_cdc_smoke() -> TestResult<()> { snapshot_number_of_parallel_tables: None, sync_interval_seconds: None, use_compression: None, + use_json_native_format: None, }, table_mappings: vec![ClickPipeMySQLPipeTableMapping { source_schema_name: "smoke_db".to_string(), diff --git a/crates/clickhouse-cloud-api/tests/clickpipes/stages/mysql.rs b/crates/clickhouse-cloud-api/tests/clickpipes/stages/mysql.rs index 5d4b2e26..64828959 100644 --- a/crates/clickhouse-cloud-api/tests/clickpipes/stages/mysql.rs +++ b/crates/clickhouse-cloud-api/tests/clickpipes/stages/mysql.rs @@ -278,6 +278,7 @@ fn build_pipe_request( snapshot_number_of_parallel_tables: None, sync_interval_seconds: None, use_compression: None, + use_json_native_format: None, }, table_mappings: vec![ClickPipeMySQLPipeTableMapping { // For MySQL, `sourceSchemaName` is the database name. diff --git a/crates/clickhouse-cloud-api/tests/clickpipes/stages/postgres.rs b/crates/clickhouse-cloud-api/tests/clickpipes/stages/postgres.rs index d71ca6ff..314e15ba 100644 --- a/crates/clickhouse-cloud-api/tests/clickpipes/stages/postgres.rs +++ b/crates/clickhouse-cloud-api/tests/clickpipes/stages/postgres.rs @@ -414,6 +414,7 @@ async fn run_variant( snapshot_num_rows_per_partition: None, snapshot_number_of_parallel_tables: None, sync_interval_seconds: None, + use_json_native_format: None, }, table_mappings: variant .mappings diff --git a/crates/clickhouse-cloud-api/tests/models/clickpipes.rs b/crates/clickhouse-cloud-api/tests/models/clickpipes.rs index c0847918..32348d0a 100644 --- a/crates/clickhouse-cloud-api/tests/models/clickpipes.rs +++ b/crates/clickhouse-cloud-api/tests/models/clickpipes.rs @@ -335,6 +335,7 @@ fn serialize_clickpipe_mutate_postgres_source_omits_absent_credentials() { snapshot_num_rows_per_partition: None, snapshot_number_of_parallel_tables: None, sync_interval_seconds: None, + use_json_native_format: None, }, skip_cert_verification: false, table_mappings: Vec::new(), @@ -2521,3 +2522,57 @@ fn clickpipe_scaling_new_sends_only_required_fields() { serde_json::json!({"replicas": 2, "replicaCpuMillicores": 500, "replicaMemoryGb": 1.5}) ); } + +#[test] +fn cdc_pipe_settings_send_use_json_native_format_only_when_set() { + let postgres = ClickPipePostgresPipeSettings { + allow_nullable_columns: false, + delete_on_merge: false, + enable_failover_slots: false, + initial_load_parallelism: None, + publication_name: None, + pull_batch_size: None, + replication_mode: ClickPipePostgresPipeSettingsReplicationmode::Cdc, + replication_slot_name: None, + snapshot_num_rows_per_partition: None, + snapshot_number_of_parallel_tables: None, + sync_interval_seconds: None, + use_json_native_format: None, + }; + let unset = serde_json::to_value(&postgres).unwrap(); + assert!(unset.get("useJsonNativeFormat").is_none()); + let set = serde_json::to_value(ClickPipePostgresPipeSettings { + use_json_native_format: Some(true), + ..postgres + }) + .unwrap(); + assert_eq!(set["useJsonNativeFormat"], serde_json::json!(true)); + + let mysql = ClickPipeMySQLPipeSettings { + allow_nullable_columns: None, + delete_on_merge: None, + initial_load_parallelism: None, + pull_batch_size: None, + replication_mechanism: None, + replication_mode: ClickPipeMySQLPipeSettingsReplicationmode::Cdc, + snapshot_num_rows_per_partition: None, + snapshot_number_of_parallel_tables: None, + sync_interval_seconds: None, + use_compression: None, + use_json_native_format: Some(false), + }; + assert_eq!( + serde_json::to_value(&mysql).unwrap()["useJsonNativeFormat"], + serde_json::json!(false) + ); +} + +#[test] +fn cdc_pipe_settings_responses_read_use_json_native_format() { + let postgres: ClickPipePostgresPipeSettingsResponse = + serde_json::from_value(serde_json::json!({"useJsonNativeFormat": true})).unwrap(); + assert_eq!(postgres.use_json_native_format, Some(true)); + let mysql: ClickPipeMySQLPipeSettingsResponse = + serde_json::from_value(serde_json::json!({"useJsonNativeFormat": null})).unwrap(); + assert_eq!(mysql.use_json_native_format, None); +} diff --git a/crates/clickhouse-openapi-analyzer/src/config.rs b/crates/clickhouse-openapi-analyzer/src/config.rs index d204243e..6876bf61 100644 --- a/crates/clickhouse-openapi-analyzer/src/config.rs +++ b/crates/clickhouse-openapi-analyzer/src/config.rs @@ -148,6 +148,12 @@ const OPTIONALITY_EXEMPTIONS: &[(&str, &str)] = &[ // make AWS create impossible in those orgs. Omission is the only valid input. // https://github.com/ClickHouse/clickhousectl/issues/1245 ("AwsBackupBucketPostRequestV1", "iamRoleExternalId"), + // useJsonNativeFormat is create-only and must be enabled for the + // organization. ClickPipePostgresPipeSettings has no required[], so the + // description fallback marks it required, which would send `false` from + // every caller, including organizations without the feature; as with + // iamRoleExternalId, omission must stay possible. Not yet verified live. + ("ClickPipePostgresPipeSettings", "useJsonNativeFormat"), // Kinesis requires protobufSchema only for Protobuf and forbids it for // other formats; the schema has no required[], and the legacy requiredness // heuristic misses that condition. diff --git a/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs b/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs index 7e1cfebe..82b6ff24 100644 --- a/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs +++ b/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs @@ -339,6 +339,7 @@ fn build_mysql_request( snapshot_number_of_parallel_tables: args.snapshot_parallel_tables, sync_interval_seconds: args.sync_interval_seconds, use_compression: args.use_compression, + use_json_native_format: None, }, table_mappings, }; diff --git a/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs b/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs index 1f8c627e..b4621cf2 100644 --- a/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs +++ b/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs @@ -290,6 +290,7 @@ fn build_postgres_pipe_settings( allow_nullable_columns: args.allow_nullable_columns.unwrap_or(false), delete_on_merge: args.delete_on_merge.unwrap_or(false), enable_failover_slots: args.enable_failover_slots.unwrap_or(false), + use_json_native_format: None, }, ) }