Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions crates/clickhouse-cloud-api/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,14 @@ from the response type for writing a fetched list back. Get needs
`control-plane:postgres-service:view`; the others need
`control-plane:postgres-service:manage`.

`ClickPipeMySQLPipeSettings`, `ClickPipePostgresPipeSettings` and their
`*Response` types gain optional `use_json_native_format`
(`useJsonNativeFormat`). `Some(true)` stores JSON columns in native
ClickHouse JSON format (ClickHouse 25.3+) instead of `String`; for Postgres,
`json[]` and `jsonb[]` become `Array(JSON)`. It can be set only on create, and
only in organizations with the feature enabled, so `None` omits it.
Struct-literal callers need `use_json_native_format: None`.

## 0.5.1

No changes to this crate; released in lockstep with `clickhousectl` 0.5.1.
Expand Down
12 changes: 11 additions & 1 deletion crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"version": "1.0",
"contact": {
"name": "ClickHouse Support",
"url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-1443100",
"url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-1443919",
"email": "support@clickhouse.com"
}
},
Expand Down Expand Up @@ -30456,6 +30456,11 @@
"description": "Enable hard delete behavior in ReplacingMergeTree for PostgreSQL DELETE operations.",
"type": "boolean",
"example": false
},
"useJsonNativeFormat": {
"description": "Store json and jsonb columns in native ClickHouse JSON format (requires ClickHouse 25.3+). Non-object values are wrapped as {\"_value\": ...}, and json[]/jsonb[] become Array(JSON). When disabled, JSON data is stored as String. Can only be set at creation. Must be enabled for your organization.",
"type": "boolean",
"example": false
}
}
},
Expand Down Expand Up @@ -30931,6 +30936,11 @@
"description": "Enable hard delete behavior in ReplacingMergeTree for MySQL DELETE operations.",
"type": "boolean",
"example": false
},
"useJsonNativeFormat": {
"description": "Store JSON columns in native ClickHouse JSON format (requires ClickHouse 25.3+). Non-object values are wrapped as {\"_value\": ...}. When disabled, JSON data is stored as String. Can only be set at creation. Must be enabled for your organization.",
"type": "boolean",
"example": false
}
},
"required": [
Expand Down
13 changes: 13 additions & 0 deletions crates/clickhouse-cloud-api/src/models/clickpipes/mysql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,14 @@ pub struct ClickPipeMySQLPipeSettings {
pub sync_interval_seconds: Option<i64>,
#[serde(rename = "useCompression", skip_serializing_if = "Option::is_none")]
pub use_compression: Option<bool>,
/// Store JSON columns in native ClickHouse JSON format (ClickHouse 25.3+);
/// otherwise they are stored as `String`. Create only, and only for
/// organizations with the feature enabled.
#[serde(
rename = "useJsonNativeFormat",
skip_serializing_if = "Option::is_none"
)]
pub use_json_native_format: Option<bool>,
}

/// `ClickPipeMySQLPipeSettings` from the ClickHouse Cloud API, in response
Expand Down Expand Up @@ -388,6 +396,11 @@ pub struct ClickPipeMySQLPipeSettingsResponse {
pub sync_interval_seconds: Option<i64>,
#[serde(rename = "useCompression", skip_serializing_if = "Option::is_none")]
pub use_compression: Option<bool>,
#[serde(
rename = "useJsonNativeFormat",
skip_serializing_if = "Option::is_none"
)]
pub use_json_native_format: Option<bool>,
}

/// `ClickPipeMySQLPipeTableMapping` from the ClickHouse Cloud API.
Expand Down
13 changes: 13 additions & 0 deletions crates/clickhouse-cloud-api/src/models/clickpipes/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,14 @@ pub struct ClickPipePostgresPipeSettings {
skip_serializing_if = "Option::is_none"
)]
pub sync_interval_seconds: Option<i64>,
/// Store JSON columns in native ClickHouse JSON format (ClickHouse 25.3+);
/// otherwise they are stored as `String`. Create only, and only for
/// organizations with the feature enabled.
#[serde(
rename = "useJsonNativeFormat",
skip_serializing_if = "Option::is_none"
)]
pub use_json_native_format: Option<bool>,
}

/// `ClickPipePostgresPipeSettings` from the ClickHouse Cloud API, in response
Expand Down Expand Up @@ -422,6 +430,11 @@ pub struct ClickPipePostgresPipeSettingsResponse {
skip_serializing_if = "Option::is_none"
)]
pub sync_interval_seconds: Option<i64>,
#[serde(
rename = "useJsonNativeFormat",
skip_serializing_if = "Option::is_none"
)]
pub use_json_native_format: Option<bool>,
}

/// `ClickPipePostgresPipeTableMapping` from the ClickHouse Cloud API.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,7 @@ async fn cloud_clickpipe_create_mysql_cdc_smoke() -> TestResult<()> {
snapshot_number_of_parallel_tables: None,
sync_interval_seconds: None,
use_compression: None,
use_json_native_format: None,
},
table_mappings: vec![ClickPipeMySQLPipeTableMapping {
source_schema_name: "smoke_db".to_string(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ fn build_pipe_request(
snapshot_number_of_parallel_tables: None,
sync_interval_seconds: None,
use_compression: None,
use_json_native_format: None,
},
table_mappings: vec![ClickPipeMySQLPipeTableMapping {
// For MySQL, `sourceSchemaName` is the database name.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,7 @@ async fn run_variant(
snapshot_num_rows_per_partition: None,
snapshot_number_of_parallel_tables: None,
sync_interval_seconds: None,
use_json_native_format: None,
},
table_mappings: variant
.mappings
Expand Down
55 changes: 55 additions & 0 deletions crates/clickhouse-cloud-api/tests/models/clickpipes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ fn serialize_clickpipe_mutate_postgres_source_omits_absent_credentials() {
snapshot_num_rows_per_partition: None,
snapshot_number_of_parallel_tables: None,
sync_interval_seconds: None,
use_json_native_format: None,
},
skip_cert_verification: false,
table_mappings: Vec::new(),
Expand Down Expand Up @@ -2521,3 +2522,57 @@ fn clickpipe_scaling_new_sends_only_required_fields() {
serde_json::json!({"replicas": 2, "replicaCpuMillicores": 500, "replicaMemoryGb": 1.5})
);
}

#[test]
fn cdc_pipe_settings_send_use_json_native_format_only_when_set() {
let postgres = ClickPipePostgresPipeSettings {
allow_nullable_columns: false,
delete_on_merge: false,
enable_failover_slots: false,
initial_load_parallelism: None,
publication_name: None,
pull_batch_size: None,
replication_mode: ClickPipePostgresPipeSettingsReplicationmode::Cdc,
replication_slot_name: None,
snapshot_num_rows_per_partition: None,
snapshot_number_of_parallel_tables: None,
sync_interval_seconds: None,
use_json_native_format: None,
};
let unset = serde_json::to_value(&postgres).unwrap();
assert!(unset.get("useJsonNativeFormat").is_none());
let set = serde_json::to_value(ClickPipePostgresPipeSettings {
use_json_native_format: Some(true),
..postgres
})
.unwrap();
assert_eq!(set["useJsonNativeFormat"], serde_json::json!(true));

let mysql = ClickPipeMySQLPipeSettings {
allow_nullable_columns: None,
delete_on_merge: None,
initial_load_parallelism: None,
pull_batch_size: None,
replication_mechanism: None,
replication_mode: ClickPipeMySQLPipeSettingsReplicationmode::Cdc,
snapshot_num_rows_per_partition: None,
snapshot_number_of_parallel_tables: None,
sync_interval_seconds: None,
use_compression: None,
use_json_native_format: Some(false),
};
assert_eq!(
serde_json::to_value(&mysql).unwrap()["useJsonNativeFormat"],
serde_json::json!(false)
);
}

#[test]
fn cdc_pipe_settings_responses_read_use_json_native_format() {
let postgres: ClickPipePostgresPipeSettingsResponse =
serde_json::from_value(serde_json::json!({"useJsonNativeFormat": true})).unwrap();
assert_eq!(postgres.use_json_native_format, Some(true));
let mysql: ClickPipeMySQLPipeSettingsResponse =
serde_json::from_value(serde_json::json!({"useJsonNativeFormat": null})).unwrap();
assert_eq!(mysql.use_json_native_format, None);
}
6 changes: 6 additions & 0 deletions crates/clickhouse-openapi-analyzer/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,12 @@ const OPTIONALITY_EXEMPTIONS: &[(&str, &str)] = &[
// make AWS create impossible in those orgs. Omission is the only valid input.
// https://github.com/ClickHouse/clickhousectl/issues/1245
("AwsBackupBucketPostRequestV1", "iamRoleExternalId"),
// useJsonNativeFormat is create-only and must be enabled for the
// organization. ClickPipePostgresPipeSettings has no required[], so the
// description fallback marks it required, which would send `false` from
// every caller, including organizations without the feature; as with
// iamRoleExternalId, omission must stay possible. Not yet verified live.
("ClickPipePostgresPipeSettings", "useJsonNativeFormat"),
// Kinesis requires protobufSchema only for Protobuf and forbids it for
// other formats; the schema has no required[], and the legacy requiredness
// heuristic misses that condition.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,7 @@ fn build_mysql_request(
snapshot_number_of_parallel_tables: args.snapshot_parallel_tables,
sync_interval_seconds: args.sync_interval_seconds,
use_compression: args.use_compression,
use_json_native_format: None,
},
table_mappings,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ fn build_postgres_pipe_settings(
allow_nullable_columns: args.allow_nullable_columns.unwrap_or(false),
delete_on_merge: args.delete_on_merge.unwrap_or(false),
enable_failover_slots: args.enable_failover_slots.unwrap_or(false),
use_json_native_format: None,
},
)
}
Expand Down
Loading