From e61067452d7393174a97eba8a3b5ff9c03e4c9be Mon Sep 17 00:00:00 2001 From: sdairs Date: Fri, 13 Nov 2026 12:28:39 +0000 Subject: [PATCH] build(cli): package only crate sources in clickhousectl and classify ClickPipes live-test paths (#1442) Restrict the published clickhousectl crate to src/** and LICENSE (Cargo still adds the root README as README.md). The live tests include the API library's harness through #[path] outside the crate, so shipping tests/ broke cargo test --no-run and clippy --all-targets on the unpacked crate. Test CLI now checks the cargo package --list contents. Map the CLI sources that live_clickpipe_postgres_cdc_test drives through `cloud clickpipe create postgres` (create.rs, destination.rs, table_mapping.rs, sources/mod.rs, sources/postgres.rs) to the clickpipes suite, matching services/query.rs and query_key.rs for the service suite. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-cli.yml | 13 ++++++++++++ crates/clickhousectl/Cargo.toml | 3 +++ docs/development.md | 2 +- scripts/classify-cloud-integration.py | 16 ++++++++++++++ .../tests/test_classify_cloud_integration.py | 21 +++++++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-cli.yml b/.github/workflows/test-cli.yml index fd5a2816..3e73410d 100644 --- a/.github/workflows/test-cli.yml +++ b/.github/workflows/test-cli.yml @@ -58,3 +58,16 @@ jobs: - name: Test run: cargo test -p clickhousectl + + # The live tests include the API library's harness by a path outside + # this crate, so the published crate must ship its sources only. + - name: Check packaged files + run: | + files=$(cargo package -p clickhousectl --list --locked) + for required in Cargo.toml LICENSE README.md src/main.rs; do + grep -qx "$required" <<<"$files" || { echo "missing from package: $required"; exit 1; } + done + if grep -v -e '^src/' -e '^Cargo\.' -e '^\.cargo_vcs_info\.json$' -e '^LICENSE$' -e '^README\.md$' <<<"$files"; then + echo "unexpected files in package (listed above)" + exit 1 + fi diff --git a/crates/clickhousectl/Cargo.toml b/crates/clickhousectl/Cargo.toml index 77df89e7..f066db2a 100644 --- a/crates/clickhousectl/Cargo.toml +++ b/crates/clickhousectl/Cargo.toml @@ -10,6 +10,9 @@ homepage.workspace = true keywords = ["clickhouse", "cli", "database", "cloud"] categories = ["command-line-utilities", "database"] readme = "../../README.md" +# Ship the CLI sources only: the tests stay in the repository, and the live +# tests reach into the API library's test harness, outside this crate. +include = ["src/**", "LICENSE"] [package.metadata.binstall] pkg-url = "https://builds.clickhouse.com/clickhousectl/{ name }-{ target }-v{ version }{ archive-suffix }" diff --git a/docs/development.md b/docs/development.md index f118b809..d3b4a7b6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -194,7 +194,7 @@ The suites load AWS credentials through the default provider chain: environment All require `CLICKHOUSE_CLOUD_API_KEY`, `CLICKHOUSE_CLOUD_API_SECRET`, `CLICKHOUSE_CLOUD_TEST_ORG_ID`, `CLICKHOUSE_CLOUD_TEST_PROVIDER`, and `CLICKHOUSE_CLOUD_TEST_REGION` in the environment, and are wired into the scheduled `Cloud Integration` GitHub Actions workflow. The ClickPipes E2E suites additionally need AWS credentials and an `eu-west-1` region quota; `clickpipe_smoke_test` reads a pre-provisioned service ID from `CLICKHOUSE_CLOUD_TEST_CLICKPIPE_SERVICE_ID`. -Two live tests drive the real `clickhousectl` binary, so they live in the CLI crate as `crates/clickhousectl/tests/live_*_test.rs`. Cargo builds the binary for them, and they share the library's harness in `crates/clickhouse-cloud-api/tests/common/` through `#[path]`. They need the same Cloud environment variables, are `#[ignore]`d, and assert JSON error codes, exit codes and Cloud API state rather than CLI wording: +Two live tests drive the real `clickhousectl` binary, so they live in the CLI crate as `crates/clickhousectl/tests/live_*_test.rs`. Cargo builds the binary for them, and they share the library's harness in `crates/clickhouse-cloud-api/tests/common/` through `#[path]`. They need the same Cloud environment variables, are `#[ignore]`d, and assert JSON error codes, exit codes and Cloud API state rather than CLI wording. The published `clickhousectl` crate ships only `src/**` (plus `LICENSE` and the root README), because those `#[path]` includes point outside the crate. The CLI sources each test drives also select its live suite: `services/query.rs` and `services/query_key.rs` select `service`, and the `clickpipe create postgres` path (`clickpipes/create.rs`, `destination.rs`, `table_mapping.rs`, `sources/mod.rs`, `sources/postgres.rs`) selects `clickpipes`: ```bash cargo test -p clickhousectl --test live_service_query_bind_test -- --ignored --nocapture # service query first-use bind diff --git a/scripts/classify-cloud-integration.py b/scripts/classify-cloud-integration.py index 53c71401..4cf55783 100644 --- a/scripts/classify-cloud-integration.py +++ b/scripts/classify-cloud-integration.py @@ -24,6 +24,22 @@ "crates/clickhousectl/src/cloud/query_api_endpoints.rs": NO_SUITES, "crates/clickhousectl/src/cloud/saved_queries.rs": NO_SUITES, "crates/clickhousectl/src/cloud/services/jwt_provider.rs": NO_SUITES, + # `live_clickpipe_postgres_cdc_test` drives `cloud clickpipe create + # postgres` live: the Postgres source, its shared auth helpers, and the + # create, destination and table-mapping builders that request uses. + "crates/clickhousectl/src/cloud/clickpipes/create.rs": frozenset({"clickpipes"}), + "crates/clickhousectl/src/cloud/clickpipes/destination.rs": frozenset( + {"clickpipes"} + ), + "crates/clickhousectl/src/cloud/clickpipes/sources/mod.rs": frozenset( + {"clickpipes"} + ), + "crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs": frozenset( + {"clickpipes"} + ), + "crates/clickhousectl/src/cloud/clickpipes/table_mapping.rs": frozenset( + {"clickpipes"} + ), # `live_service_query_bind_test` drives `cloud service query` live. "crates/clickhousectl/src/cloud/services/query.rs": frozenset({"service"}), "crates/clickhousectl/src/cloud/services/query_key.rs": frozenset({"service"}), diff --git a/scripts/tests/test_classify_cloud_integration.py b/scripts/tests/test_classify_cloud_integration.py index b970c202..c856d4d5 100644 --- a/scripts/tests/test_classify_cloud_integration.py +++ b/scripts/tests/test_classify_cloud_integration.py @@ -38,6 +38,13 @@ def test_path_mapping(self): "crates/clickhouse-cloud-api/tests/service_query_key_cli_test.rs": frozenset( {"service"} ), + "crates/clickhousectl/src/cloud/clickpipes/create.rs": frozenset( + {"clickpipes"} + ), + "crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs": frozenset( + {"clickpipes"} + ), + "crates/clickhousectl/src/cloud/clickpipes/manage.rs": classifier.NO_SUITES, "Cargo.lock": classifier.ALL_SUITES, "crates/clickhousectl/Cargo.toml": classifier.ALL_SUITES, ".cargo/config.toml": classifier.ALL_SUITES, @@ -192,6 +199,20 @@ def test_udf_paths_select_service_lifecycle(self): self.assertEqual(selection.suites, ("service",)) self.assertFalse(selection.failed_closed) + def test_live_cli_clickpipe_create_postgres_paths_select_clickpipes(self): + for path in ( + "crates/clickhousectl/src/cloud/clickpipes/create.rs", + "crates/clickhousectl/src/cloud/clickpipes/destination.rs", + "crates/clickhousectl/src/cloud/clickpipes/sources/mod.rs", + "crates/clickhousectl/src/cloud/clickpipes/sources/postgres.rs", + "crates/clickhousectl/src/cloud/clickpipes/table_mapping.rs", + ): + with self.subTest(path=path): + self.assertTrue((classifier.REPO_ROOT / path).is_file()) + selection = classifier.select_records([("M", (path,))]) + self.assertEqual(selection.suites, ("clickpipes",)) + self.assertFalse(selection.failed_closed) + def test_every_current_cloud_api_test_file_has_an_explicit_mapping(self): test_root = ( classifier.REPO_ROOT / "crates" / "clickhouse-cloud-api" / "tests"