diff --git a/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs b/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs index 82b6ff24..d3a11d82 100644 --- a/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs +++ b/crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs @@ -172,6 +172,13 @@ pub struct MySqlCreateArgs { #[arg(long, value_name = "true|false")] pub use_compression: Option, + /// Store JSON columns as native ClickHouse JSON (create-time only) + /// + /// Needs ClickHouse 25.3+ and the feature enabled for the organization; + /// otherwise JSON is stored as String. + #[arg(long, value_name = "true|false")] + pub use_json_native_format: Option, + #[command(flatten)] pub destination: DatabaseDestinationArgs, @@ -339,7 +346,7 @@ fn build_mysql_request( snapshot_number_of_parallel_tables: args.snapshot_parallel_tables, sync_interval_seconds: args.sync_interval_seconds, use_compression: args.use_compression, - use_json_native_format: None, + use_json_native_format: args.use_json_native_format, }, table_mappings, }; @@ -450,6 +457,8 @@ mod tests { "true", "--use-compression", "false", + "--use-json-native-format", + "true", "--destination-database", "analytics", "--org-id", @@ -485,6 +494,7 @@ mod tests { assert_eq!(args.allow_nullable_columns, Some(false)); assert_eq!(args.delete_on_merge, Some(true)); assert_eq!(args.use_compression, Some(false)); + assert_eq!(args.use_json_native_format, Some(true)); assert_eq!(args.destination.destination_database, "analytics"); let ClickPipeCommands::Create { @@ -532,6 +542,7 @@ mod tests { assert_eq!(args.allow_nullable_columns, None); assert_eq!(args.delete_on_merge, None); assert_eq!(args.use_compression, None); + assert_eq!(args.use_json_native_format, None); assert_eq!(args.destination.destination_database, "default"); for invalid in ["0", "4294967296"] { @@ -780,6 +791,7 @@ mod tests { allow_nullable_columns: None, delete_on_merge: None, use_compression: None, + use_json_native_format: None, destination: DatabaseDestinationArgs { destination_database: "default".into(), }, @@ -830,6 +842,7 @@ mod tests { assert_eq!(source.settings.snapshot_number_of_parallel_tables, None); assert_eq!(source.settings.sync_interval_seconds, None); assert_eq!(source.settings.use_compression, None); + assert_eq!(source.settings.use_json_native_format, None); assert_eq!(source.table_mappings.len(), 1); assert_eq!(source.table_mappings[0].source_schema_name, "source"); assert_eq!(source.table_mappings[0].source_table, "events"); @@ -869,6 +882,7 @@ mod tests { args.allow_nullable_columns = Some(false); args.delete_on_merge = Some(true); args.use_compression = Some(false); + args.use_json_native_format = Some(true); args.destination.destination_database = "analytics".into(); args.destination_roles = DestinationRoleArgs { roles: vec!["analytics_reader".into()], @@ -919,6 +933,7 @@ mod tests { assert_eq!(source.settings.allow_nullable_columns, Some(false)); assert_eq!(source.settings.delete_on_merge, Some(true)); assert_eq!(source.settings.use_compression, Some(false)); + assert_eq!(source.settings.use_json_native_format, Some(true)); assert_eq!(source.table_mappings.len(), 2); let mapping = &source.table_mappings[1]; assert_eq!(mapping.source_schema_name, "sales"); diff --git a/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs b/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs index b4621cf2..3ff253bc 100644 --- a/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs +++ b/crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs @@ -187,6 +187,13 @@ pub struct PostgresCreateArgs { #[arg(long, value_name = "true|false")] pub delete_on_merge: Option, + /// Store json/jsonb as native ClickHouse JSON (create-time only) + /// + /// Needs ClickHouse 25.3+ and the feature enabled for the organization; + /// otherwise JSON is stored as String. + #[arg(long, value_name = "true|false")] + pub use_json_native_format: Option, + #[command(flatten)] pub destination: DatabaseDestinationArgs, @@ -290,7 +297,7 @@ fn build_postgres_pipe_settings( allow_nullable_columns: args.allow_nullable_columns.unwrap_or(false), delete_on_merge: args.delete_on_merge.unwrap_or(false), enable_failover_slots: args.enable_failover_slots.unwrap_or(false), - use_json_native_format: None, + use_json_native_format: args.use_json_native_format, }, ) } @@ -538,6 +545,7 @@ mod tests { assert_eq!(args.allow_nullable_columns, None); assert_eq!(args.enable_failover_slots, None); assert_eq!(args.delete_on_merge, None); + assert_eq!(args.use_json_native_format, None); assert_eq!(args.destination.destination_database, "default"); } @@ -561,6 +569,8 @@ mod tests { "false", "--delete-on-merge", "true", + "--use-json-native-format", + "false", ]); let ClickPipeCommands::Create { command: ClickPipeCreateCommands::Postgres(args), @@ -579,6 +589,7 @@ mod tests { assert_eq!(args.allow_nullable_columns, Some(true)); assert_eq!(args.enable_failover_slots, Some(false)); assert_eq!(args.delete_on_merge, Some(true)); + assert_eq!(args.use_json_native_format, Some(false)); } #[test] @@ -1053,6 +1064,7 @@ mod tests { allow_nullable_columns: None, enable_failover_slots: None, delete_on_merge: None, + use_json_native_format: None, destination: DatabaseDestinationArgs { destination_database: "default".into(), }, @@ -1103,6 +1115,8 @@ mod tests { assert!(!source.settings.allow_nullable_columns); assert!(!source.settings.delete_on_merge); assert!(!source.settings.enable_failover_slots); + // Unset stays omitted, so organizations without the feature never send it. + assert_eq!(source.settings.use_json_native_format, None); assert_eq!(source.settings.initial_load_parallelism, None); assert_eq!(source.settings.pull_batch_size, None); assert_eq!(source.settings.snapshot_num_rows_per_partition, None); @@ -1155,6 +1169,7 @@ mod tests { args.allow_nullable_columns = Some(true); args.enable_failover_slots = Some(true); args.delete_on_merge = Some(true); + args.use_json_native_format = Some(true); args.destination.destination_database = "analytics".into(); args.destination_roles = DestinationRoleArgs { roles: vec!["analytics_reader".into(), "analytics_writer".into()], @@ -1206,6 +1221,7 @@ mod tests { assert!(source.settings.allow_nullable_columns); assert!(source.settings.enable_failover_slots); assert!(source.settings.delete_on_merge); + assert_eq!(source.settings.use_json_native_format, Some(true)); assert_eq!(source.table_mappings.len(), 2); assert_eq!(source.table_mappings[0].source_schema_name, "public"); assert_eq!(source.table_mappings[0].source_table, "users"); diff --git a/docs/reference/clickpipe-configuration.md b/docs/reference/clickpipe-configuration.md index d62c8ae2..3b0e88b5 100644 --- a/docs/reference/clickpipe-configuration.md +++ b/docs/reference/clickpipe-configuration.md @@ -85,7 +85,7 @@ Supported engines are `MergeTree`, `ReplacingMergeTree`, and `Null`. Postgres de Kafka, Kinesis, object storage, and Pub/Sub support `--start-paused`, repeatable JSON `--field-mapping`, and initial scaling. Supply all three of `--replicas`, `--cpu-millicores`, and `--memory-gb` together for an allocation. Database CDC scaling is service-wide via `clickpipe cdc-scaling`, not per-pipe allocation. `cdc-scaling update` takes both `--cpu-millicores` and `--memory-gb`, with memory GiB equal to millicores / 250 (for example `--cpu-millicores 2000 --memory-gb 8`). It returns once the change is accepted and prints the old allocation; it applies within minutes, so poll `cdc-scaling get` until the new values show. Another scaling update before then fails with a conflict. -For Postgres, snapshot worker/partition settings, nullability, failover slots, and delete-on-merge are create-time choices. Only `syncIntervalSeconds` and `pullBatchSize` can be patched later. `--replication-slot-name` requires `--replication-mode cdc_only`; PG17+ failover slots apply when ClickPipes creates the slot, not when reusing one. MongoDB `--initial-load-parallelism` is per collection; `--snapshot-parallel-collections` controls concurrent collections. +For Postgres, snapshot worker/partition settings, nullability, failover slots, and delete-on-merge are create-time choices. Only `syncIntervalSeconds` and `pullBatchSize` can be patched later. `--replication-slot-name` requires `--replication-mode cdc_only`; PG17+ failover slots apply when ClickPipes creates the slot, not when reusing one. MongoDB `--initial-load-parallelism` is per collection; `--snapshot-parallel-collections` controls concurrent collections. For Postgres and MySQL, `--use-json-native-format true` stores JSON columns as native ClickHouse `JSON` instead of `String`, also a create-time choice. It needs ClickHouse 25.3+ and the feature enabled for the organization; non-object values are wrapped as `{"_value": ...}`, and Postgres `json[]`/`jsonb[]` become `Array(JSON)`. Kafka/Kinesis Protobuf schema inputs accept `.proto` or serialized `FileDescriptorSet`; use `--protobuf-schema-file -` for stdin. The API limits the encoded schema to 1 MiB. Kafka Protobuf input conflicts with registry flags; `--exactly-once` is create-only. Pub/Sub timestamp seek requires `--seek-timestamp`; `earliest` reads backlog and `latest` reads new messages only.