Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion crates/clickhousectl/src/cloud/clickpipes/sources/mysql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,13 @@ pub struct MySqlCreateArgs {
#[arg(long, value_name = "true|false")]
pub use_compression: Option<bool>,

/// Store JSON columns as native ClickHouse JSON (create-time only)
///
/// Needs ClickHouse 25.3+ and the feature enabled for the organization;
/// otherwise JSON is stored as String.
#[arg(long, value_name = "true|false")]
pub use_json_native_format: Option<bool>,

#[command(flatten)]
pub destination: DatabaseDestinationArgs,

Expand Down Expand Up @@ -339,7 +346,7 @@ fn build_mysql_request(
snapshot_number_of_parallel_tables: args.snapshot_parallel_tables,
sync_interval_seconds: args.sync_interval_seconds,
use_compression: args.use_compression,
use_json_native_format: None,
use_json_native_format: args.use_json_native_format,
},
table_mappings,
};
Expand Down Expand Up @@ -450,6 +457,8 @@ mod tests {
"true",
"--use-compression",
"false",
"--use-json-native-format",
"true",
"--destination-database",
"analytics",
"--org-id",
Expand Down Expand Up @@ -485,6 +494,7 @@ mod tests {
assert_eq!(args.allow_nullable_columns, Some(false));
assert_eq!(args.delete_on_merge, Some(true));
assert_eq!(args.use_compression, Some(false));
assert_eq!(args.use_json_native_format, Some(true));
assert_eq!(args.destination.destination_database, "analytics");

let ClickPipeCommands::Create {
Expand Down Expand Up @@ -532,6 +542,7 @@ mod tests {
assert_eq!(args.allow_nullable_columns, None);
assert_eq!(args.delete_on_merge, None);
assert_eq!(args.use_compression, None);
assert_eq!(args.use_json_native_format, None);
assert_eq!(args.destination.destination_database, "default");

for invalid in ["0", "4294967296"] {
Expand Down Expand Up @@ -780,6 +791,7 @@ mod tests {
allow_nullable_columns: None,
delete_on_merge: None,
use_compression: None,
use_json_native_format: None,
destination: DatabaseDestinationArgs {
destination_database: "default".into(),
},
Expand Down Expand Up @@ -830,6 +842,7 @@ mod tests {
assert_eq!(source.settings.snapshot_number_of_parallel_tables, None);
assert_eq!(source.settings.sync_interval_seconds, None);
assert_eq!(source.settings.use_compression, None);
assert_eq!(source.settings.use_json_native_format, None);
assert_eq!(source.table_mappings.len(), 1);
assert_eq!(source.table_mappings[0].source_schema_name, "source");
assert_eq!(source.table_mappings[0].source_table, "events");
Expand Down Expand Up @@ -869,6 +882,7 @@ mod tests {
args.allow_nullable_columns = Some(false);
args.delete_on_merge = Some(true);
args.use_compression = Some(false);
args.use_json_native_format = Some(true);
args.destination.destination_database = "analytics".into();
args.destination_roles = DestinationRoleArgs {
roles: vec!["analytics_reader".into()],
Expand Down Expand Up @@ -919,6 +933,7 @@ mod tests {
assert_eq!(source.settings.allow_nullable_columns, Some(false));
assert_eq!(source.settings.delete_on_merge, Some(true));
assert_eq!(source.settings.use_compression, Some(false));
assert_eq!(source.settings.use_json_native_format, Some(true));
assert_eq!(source.table_mappings.len(), 2);
let mapping = &source.table_mappings[1];
assert_eq!(mapping.source_schema_name, "sales");
Expand Down
18 changes: 17 additions & 1 deletion crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,13 @@ pub struct PostgresCreateArgs {
#[arg(long, value_name = "true|false")]
pub delete_on_merge: Option<bool>,

/// Store json/jsonb as native ClickHouse JSON (create-time only)
///
/// Needs ClickHouse 25.3+ and the feature enabled for the organization;
/// otherwise JSON is stored as String.
#[arg(long, value_name = "true|false")]
pub use_json_native_format: Option<bool>,

#[command(flatten)]
pub destination: DatabaseDestinationArgs,

Expand Down Expand Up @@ -290,7 +297,7 @@ fn build_postgres_pipe_settings(
allow_nullable_columns: args.allow_nullable_columns.unwrap_or(false),
delete_on_merge: args.delete_on_merge.unwrap_or(false),
enable_failover_slots: args.enable_failover_slots.unwrap_or(false),
use_json_native_format: None,
use_json_native_format: args.use_json_native_format,
},
)
}
Expand Down Expand Up @@ -538,6 +545,7 @@ mod tests {
assert_eq!(args.allow_nullable_columns, None);
assert_eq!(args.enable_failover_slots, None);
assert_eq!(args.delete_on_merge, None);
assert_eq!(args.use_json_native_format, None);
assert_eq!(args.destination.destination_database, "default");
}

Expand All @@ -561,6 +569,8 @@ mod tests {
"false",
"--delete-on-merge",
"true",
"--use-json-native-format",
"false",
]);
let ClickPipeCommands::Create {
command: ClickPipeCreateCommands::Postgres(args),
Expand All @@ -579,6 +589,7 @@ mod tests {
assert_eq!(args.allow_nullable_columns, Some(true));
assert_eq!(args.enable_failover_slots, Some(false));
assert_eq!(args.delete_on_merge, Some(true));
assert_eq!(args.use_json_native_format, Some(false));
}

#[test]
Expand Down Expand Up @@ -1053,6 +1064,7 @@ mod tests {
allow_nullable_columns: None,
enable_failover_slots: None,
delete_on_merge: None,
use_json_native_format: None,
destination: DatabaseDestinationArgs {
destination_database: "default".into(),
},
Expand Down Expand Up @@ -1103,6 +1115,8 @@ mod tests {
assert!(!source.settings.allow_nullable_columns);
assert!(!source.settings.delete_on_merge);
assert!(!source.settings.enable_failover_slots);
// Unset stays omitted, so organizations without the feature never send it.
assert_eq!(source.settings.use_json_native_format, None);
assert_eq!(source.settings.initial_load_parallelism, None);
assert_eq!(source.settings.pull_batch_size, None);
assert_eq!(source.settings.snapshot_num_rows_per_partition, None);
Expand Down Expand Up @@ -1155,6 +1169,7 @@ mod tests {
args.allow_nullable_columns = Some(true);
args.enable_failover_slots = Some(true);
args.delete_on_merge = Some(true);
args.use_json_native_format = Some(true);
args.destination.destination_database = "analytics".into();
args.destination_roles = DestinationRoleArgs {
roles: vec!["analytics_reader".into(), "analytics_writer".into()],
Expand Down Expand Up @@ -1206,6 +1221,7 @@ mod tests {
assert!(source.settings.allow_nullable_columns);
assert!(source.settings.enable_failover_slots);
assert!(source.settings.delete_on_merge);
assert_eq!(source.settings.use_json_native_format, Some(true));
assert_eq!(source.table_mappings.len(), 2);
assert_eq!(source.table_mappings[0].source_schema_name, "public");
assert_eq!(source.table_mappings[0].source_table, "users");
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/clickpipe-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ Supported engines are `MergeTree`, `ReplacingMergeTree`, and `Null`. Postgres de

Kafka, Kinesis, object storage, and Pub/Sub support `--start-paused`, repeatable JSON `--field-mapping`, and initial scaling. Supply all three of `--replicas`, `--cpu-millicores`, and `--memory-gb` together for an allocation. Database CDC scaling is service-wide via `clickpipe cdc-scaling`, not per-pipe allocation. `cdc-scaling update` takes both `--cpu-millicores` and `--memory-gb`, with memory GiB equal to millicores / 250 (for example `--cpu-millicores 2000 --memory-gb 8`). It returns once the change is accepted and prints the old allocation; it applies within minutes, so poll `cdc-scaling get` until the new values show. Another scaling update before then fails with a conflict.

For Postgres, snapshot worker/partition settings, nullability, failover slots, and delete-on-merge are create-time choices. Only `syncIntervalSeconds` and `pullBatchSize` can be patched later. `--replication-slot-name` requires `--replication-mode cdc_only`; PG17+ failover slots apply when ClickPipes creates the slot, not when reusing one. MongoDB `--initial-load-parallelism` is per collection; `--snapshot-parallel-collections` controls concurrent collections.
For Postgres, snapshot worker/partition settings, nullability, failover slots, and delete-on-merge are create-time choices. Only `syncIntervalSeconds` and `pullBatchSize` can be patched later. `--replication-slot-name` requires `--replication-mode cdc_only`; PG17+ failover slots apply when ClickPipes creates the slot, not when reusing one. MongoDB `--initial-load-parallelism` is per collection; `--snapshot-parallel-collections` controls concurrent collections. For Postgres and MySQL, `--use-json-native-format true` stores JSON columns as native ClickHouse `JSON` instead of `String`, also a create-time choice. It needs ClickHouse 25.3+ and the feature enabled for the organization; non-object values are wrapped as `{"_value": ...}`, and Postgres `json[]`/`jsonb[]` become `Array(JSON)`.

Kafka/Kinesis Protobuf schema inputs accept `.proto` or serialized `FileDescriptorSet`; use `--protobuf-schema-file -` for stdin. The API limits the encoded schema to 1 MiB. Kafka Protobuf input conflicts with registry flags; `--exactly-once` is create-only. Pub/Sub timestamp seek requires `--seek-timestamp`; `earliest` reads backlog and `latest` reads new messages only.

Expand Down
Loading