diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1682e8f..aad9cf88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -408,12 +408,12 @@ jobs: strategy: fail-fast: false matrix: - # "1.20" = lower-bound pin from pyproject.toml `mcp>=1.20` + # "1.22" = lower-bound pin from pyproject.toml `mcp>=1.22` # (see comment there for the FastMCP API features we # need: deferred-annotation handling, ClientSession # ``sampling_capabilities`` kwarg, etc.) # "range-max" = newest release inside the pyproject constraint - # (`mcp>=1.20,<3.0`) — what a fresh + # (`mcp>=1.22,<3.0`) — what a fresh # `pip install data-product-forge` resolves today. # Since Phase 2 widened the ceiling this resolves to # 2.x, and this leg is NOT warn-wrapped, so 2.x is @@ -425,7 +425,7 @@ jobs: # warn-only, for the same reason it was warn-only # for 2.x: an upstream release must not be able to # paint every open PR red. - mcp-version: ["1.20", "range-max", "latest"] + mcp-version: ["1.22", "range-max", "latest"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -451,7 +451,7 @@ jobs: done case "${{ matrix.mcp-version }}" in # Keep the range-max constraint in sync with pyproject.toml. - range-max) pip install --upgrade "mcp>=1.20,<3.0" ;; + range-max) pip install --upgrade "mcp>=1.22,<3.0" ;; latest) pip install --upgrade mcp ;; *) pip install --force-reinstall "mcp==${{ matrix.mcp-version }}" ;; esac @@ -470,7 +470,7 @@ jobs: version = md.version("mcp") major = int(version.split(".")[0]) print(f"leg={leg} resolved mcp=={version} (major {major})") - expected = {"1.20": 1, "range-max": 2, "latest": 2}[leg] + expected = {"1.22": 1, "range-max": 2, "latest": 2}[leg] if major != expected: sys.exit( f"::error::leg '{leg}' resolved mcp {version}, expected major {expected}. " diff --git a/CHANGELOG.md b/CHANGELOG.md index 54785486..739c6a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Upgrade notes +- The minimum `mcp` is now 1.22 (was 1.20). mcp 1.20 and 1.21 import + `eval_type_backport` from pydantic's private `_typing_extra` module, which pydantic 2.14 + removed, so `mcp.server.fastmcp`, and every forge-cli MCP surface with it, failed to + import on a fresh install that resolved those releases with current pydantic. 1.22 is + the first 1.x that imports with pydantic 2.14. The CI drift leg that pins the floor moves + from 1.20 to 1.22. - An AWS contract that declared `location.catalog` other than `glue` (for example `rest`, `iceberg_rest`, `polaris`, `unity`, `nessie` or `lakekeeper`) on an Iceberg expose holds a Glue database and table in its OpenTofu state that earlier releases @@ -29,6 +35,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 table lived in two catalogs; `fluid validate` and the runners now refuse it unless the sink writes BigLake. Set `catalog: bigquery`, or the REST kind your catalog is. A hand-written sink config must then select that same catalog. +- An Iceberg expose with `catalog: dynamodb` or `catalog: jdbc` that a streaming sink + writes to now needs a warehouse: both catalogs refuse to start without one. Set + `location.warehouse`, or an explicit `location.bucket` on `platform: aws` or `gcp`, from + which forge-cli derives `s3://` or `gs://` `/` (`path` defaults to + `//`), or set the sink's `warehouse` property in an override. With none + of these, `fluid validate` and the runner preflight refuse the build. +- An Iceberg expose with `catalog: bigquery` that a streaming sink writes to now needs + `location.project`, which becomes the catalog's `gcp.bigquery.project-id`, or that + property in an override. A blank value counts as missing. +- The derived BigQuery sink config no longer sets `client.region`: `location.region` goes + to `gcp.bigquery.location`. Its warehouse is now the `gs://` storage URI dbt-bigquery and + the GCP IaC use (a `gs://` `location.warehouse`, else `gs://` plus + `location.path` when set, with the bucket's `{{ env.* }}` templates rendered), not + `location.warehouse` as written; a warehouse with another scheme, or none and no bucket, + gives none, and `fluid validate` warns. +- A `platform: confluent` Iceberg expose with no `location.catalog` now reads as Glue, the + catalog Tableflow publishes it to, and dbt-snowflake `catalogs.yml` writes + `catalog_linked_database_type: glue` for it. `fluid policy compile` grants a Confluent + Tableflow expose its S3 bucket and the Glue table Tableflow publishes: the topic's table + (`topic`, else `location.table`, else the expose id, as the Tableflow IaC names it) in + `location.database`, or, with none, in the Kafka cluster id database Tableflow defaults + to. With neither database nor `kafka_cluster_id` it warns and grants no Glue table. +- A Kafka Connect build whose Iceberg sink config forge-cli derives now writes exactly one + Iceberg expose, picked by its `outputs`. `fluid validate` and the run preflight refuse it + when the outputs name no Iceberg sink expose or name two or more, and when it declares + no outputs and the contract has two or more. Outputs that named no Iceberg sink expose + used to draw only the warning "the join is implicit". Split the build into one build per + expose, or hand-write `properties.kafka-connect.sink_connector_config`. +- An embedded Debezium Server build whose Iceberg sink config forge-cli derives now writes + the Iceberg sink exposes its `outputs` name, or all of them with no outputs. It is + refused when those exposes sit in more than one database, resolve to different catalogs, + or when the outputs name none of them. +- `fluid policy compile` now exits 1 with `policy_compiler_crashed` and writes no bindings + file when the policy compiler crashes. It used to write an empty bindings list and exit + 0, so the next stage read the crash as "no grants to enforce". +- A Kafka Connect or embedded Debezium build whose sink config forge-cli derives is now + refused when its expose names no table namespace (`location.database`, or on GCP + `location.dataset`), and a Kafka Connect one when its expose names no `location.table`. + The derived config used to name the table `None.
`, or leave Debezium Server on its + default namespace. An override that sets `iceberg.tables` or `table-namespace` itself is + not affected. ### Changed @@ -57,6 +104,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 it accepted. They ran beside `publish-pypi` and ignored its result, so a refused upload (for example "file already exists" on a re-dispatched tag) still cut a release and moved the GHCR `latest` tag. The release takes the quality gate's validated tag name. +- A derived Kafka Connect or embedded Debezium Server Iceberg sink resolves the exposes it + writes from the build's `outputs`, through one resolver that `fluid validate`, the run + preflight, both runners and the Kafka Connect late-arrival target read. The Debezium + Server sink writes every captured table under one `table-namespace` through one catalog, + so the exposes must share one `binding.location.database` and one catalog. A Glue + warehouse is a per-table prefix: it is not compared, and the sink takes the first + expose's. A DynamoDB or JDBC catalog creates every missing table under the one warehouse + the sink is given, so warehouses derived from `location.bucket` that differ (each is that + expose's own table prefix) are refused; set one `location.warehouse` on those exposes, or + split the build. A hand-written config, and an override that sets + `iceberg.tables` (Kafka Connect) or `table-namespace` (Debezium Server), is still checked + against the first Iceberg sink expose. +- The Kafka Connect deriver now forwards the catalog's own properties (for BigQuery, + `gcp.bigquery.project-id` and `gcp.bigquery.location`) under the `iceberg.catalog.` + prefix, as the Debezium Server deriver already did. ### Fixed @@ -206,6 +268,61 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a name its SQL cannot quote all passed stage 2 and failed first at `fluid generate iac` or `fluid apply`. Both now run the same derivation (`lf_governance`) and refuse with the same message. +- **A second CDC build no longer streams into the first build's table.** A Kafka Connect + or embedded Debezium build whose sink config forge-cli derives always wrote the first + Iceberg expose, so a build for `refunds` pushed `iceberg.tables=sales.orders` while + `fluid validate`, the preflight and the run all passed. It now writes the expose its + `outputs` name (see Upgrade notes). +- **A GCP streaming sink writes into the expose's BigQuery dataset.** The sink named the + table's namespace from `location.database` only, but a GCP binding names its dataset in + `location.dataset`, so a BigQuery or other GCP Iceberg sink pushed + `iceberg.tables=None.
` and `fluid validate` passed. The namespace is now + `location.database`, else `location.dataset` (the BigQuery metastore catalog maps a + namespace to a dataset), and a sink with neither is refused (see Upgrade notes). The + Kafka Connect late-events table and the Debezium shared-database check read it the + same way. +- **DynamoDB, JDBC and BigQuery sink configs carry what the catalog needs to start.** + `DynamoDbCatalog` and `JdbcCatalog` refuse to initialise without a warehouse, and + `BigQueryMetastoreCatalog` without `gcp.bigquery.project-id`; forge-cli derived neither + and `fluid validate` asked for neither, so the connector failed at start (see Upgrade + notes). A warehouse or project an override supplies counts, and a whitespace-only + `location.warehouse` or `location.project` counts as unset. +- **A bucket written as a `{{ env.* }}` template is read as written.** When its variable is + unset or empty where `fluid validate` runs, validate warns and names the variable instead + of asking for a bucket the contract already sets. The runner preflight, where the sink + config is derived, refuses the build instead of pushing a warehouse in a bucket such as + `acme--lake` that the contract does not name; this includes a run from a plan, whose + embedded contract is read as written. A Kafka Connect BigQuery sink reads the warehouse + only to auto-create tables, so it is refused only with auto-create on; otherwise it + warns and pushes no warehouse. An override warehouse is kept, and a hand-written sink + config that does not derive is not checked against the bucket's variables. +- **The BigQuery no-warehouse warning states what fails.** With auto-create on, the Kafka + Connect sink calls `createNamespace` for each table it creates, and + `BigQueryMetastoreCatalog` refuses that without a warehouse, so auto-create fails even in + a dataset that exists. For Debezium Server it says the server does not boot without + `debezium.sink.iceberg.warehouse`, which has no default. +- **`fluid policy compile` grants a Confluent Tableflow table with no catalog.** A + `platform: confluent` Iceberg expose with no catalog read as `rest`, so `fluid policy + compile` dropped its grants with a false "cataloged in 'rest'" warning (see Upgrade + notes). +- **`fluid policy compile` shows its warnings.** Each compiler warning, for example a grant + on an Iceberg table in a catalog it cannot grant on, which compiles to no table grant, is + now printed at WARNING level, and the exit code stays 0; the warnings are still written to + `bindings.json`. `fluid policy apply` echoes the warnings stored in `bindings.json`. A + contract with no grants prints no warning. +- **A policy compiler crash fails the command.** `fluid policy compile` and the policy step + of `fluid generate artifacts` now fail with exit 1 and `policy_compiler_crashed` (see + Upgrade notes). The error names the schema type errors at the values the compiler reads + (`accessPolicy`, its grants, each grant's permissions, each expose's binding and + location) by path and expected type, never by value (a value of the wrong type can be a + connection URL with a password in it), or the compiler's own exception when the schema + finds none there. +- **The Snowflake EXTERNAL VOLUME catalog-move guard names both causes.** It blocks the + same applies, but its message no longer claims forge-cli stopped + creating the volume or that the moved exposes held it. It names both possible causes (an + upgrade from a release that gave a non-Snowflake-managed catalog a volume, or a + Snowflake-managed expose removed or re-cataloged in this change) and keeps the `tofu + state rm` remediation. ### Documentation diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index f807b5d7..e013006d 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -341,6 +341,19 @@ def slug_for(event: str) -> str: ["Check the agent-policy block in the contract; run 'fluid policy check '"], "sovereignty", ), + "policy_compiler_crashed": ( + [ + ( + "Run 'fluid validate ': policy compile reads accessPolicy and " + + "exposes without validating them against the contract schema" + ), + ( + "If the contract validates, re-run with 'fluid --log-level DEBUG policy compile " + + "' to see the compiler's traceback" + ), + ], + "policy-compile#errors", + ), "policy_apply_failed": ( ["Run 'fluid policy check ' to surface the offending rule before apply"], "sovereignty", diff --git a/fluid_build/_errors.py b/fluid_build/_errors.py index dce8617c..af65a38d 100644 --- a/fluid_build/_errors.py +++ b/fluid_build/_errors.py @@ -83,6 +83,8 @@ "infra#drift": "advanced/typed-cli-errors.html#governance", # the safety gate's own section, which shows its message and remediation "iceberg-catalog-move": "cli/apply.html#iceberg-catalog-move-guard", + # the command's own Errors section + "policy-compile#errors": "cli/policy-compile.html#errors", } _DOC_FALLBACK = "advanced/production-troubleshooting.html" diff --git a/fluid_build/_mcp_compat.py b/fluid_build/_mcp_compat.py index 1648ff72..83423fed 100644 --- a/fluid_build/_mcp_compat.py +++ b/fluid_build/_mcp_compat.py @@ -35,9 +35,9 @@ ``open_inmemory_session``). This module is the ONLY place allowed to branch on the SDK generation. -Runtime code keeps the ``mcp>=1.20,<2.0`` install pin (pyproject.toml) -until the 2.x drift leg has a green track record — the dual support here -is what turns that leg green in the first place. +The install pin (pyproject.toml) is ``mcp>=1.22,<3.0``, so both +generations install — the dual support here is what keeps both drift legs +green. Tier-0 leaf rules: no module-scope ``mcp`` import (the startup budget in ``tests/perf/test_startup_budget.py`` forbids ``mcp`` on ``fluid --help``); diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index e91f80bc..835bdab8 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -203,6 +203,7 @@ def _execute_acquisition_build( *, dry_run: bool, sample_rows: Any = None, + as_written: Optional[Dict[str, Any]] = None, ) -> int: """Dispatch an acquisition build to its runner. @@ -210,6 +211,10 @@ def _execute_acquisition_build( the registered callable. Capability negotiation runs first so a contract asking for an unsupported capability fails with the rich ``CapabilityMismatchError`` instead of running the wrong shape. + + ``as_written`` is ``contract`` before any ``{{ env.* }}`` resolution, when + the caller resolved some already (a plan-embedded contract); ``None`` + means ``contract`` is as written. """ engine = (build.get("engine") or "").strip().lower() @@ -252,21 +257,32 @@ def _execute_acquisition_build( # everything (including secret-shaped vars). Catalog-export paths use # ``cli/_common.py::resolve_contract_env_templates`` instead, which # leaves sensitive placeholders literal. + unresolved_contract = contract if as_written is None else as_written build = _resolve_env_placeholders(build) contract = _resolve_env_placeholders(contract) module_path, function_name = entry + import contextlib import importlib module = importlib.import_module(module_path, package="fluid_build.build_runners") runner_fn = getattr(module, function_name) - return runner_fn( - build, - contract, - contract_dir, - dry_run=dry_run, - sample_rows=sample_rows, - ) + # The Iceberg sink preflight reads a bucket's ``{{ env.X }}`` template as + # written: the resolution above turned an unset ``X`` into "", which would + # point the sink at a bucket the contract does not name. + scope: contextlib.AbstractContextManager[None] = contextlib.nullcontext() + if engine in ("kafka-connect", "debezium"): + from .kafka_connect.iceberg_sink_validation import contract_as_written + + scope = contract_as_written(unresolved_contract) + with scope: + return runner_fn( + build, + contract, + contract_dir, + dry_run=dry_run, + sample_rows=sample_rows, + ) def _execute_embedded_sql_build_snowflake( @@ -989,6 +1005,7 @@ def run_builds_from_args( 1, "contract_load_failed", {"path": str(contract_path), "error": str(exc)} ) contract = plan_data.get("contract") or {} + contract_as_written = contract if not contract: raise CLIError( 1, @@ -1061,6 +1078,7 @@ def run_builds_from_args( raise except Exception as e: raise CLIError(1, "contract_load_failed", {"path": str(contract_path), "error": str(e)}) + contract_as_written = contract if str(contract_path).lower().endswith((".tgz", ".tar.gz")): # A bundle's own directory is not where its contract lives: # anchor at the source contract its MANIFEST records. @@ -1151,12 +1169,16 @@ def run_builds_from_args( if is_acquisition_build(build): sample_rows = getattr(args, "sample_rows", None) + # A plan's contract had its ``{{ env.* }}`` templates resolved + # above, with an empty variable rendered to "": the Iceberg sink + # preflight reads the bucket template from the plan as written. result = _execute_acquisition_build( build, contract, contract_path.parent, dry_run=args.dry_run, sample_rows=sample_rows, + as_written=contract_as_written, ) _report_build(report, contract_path.parent, product_id, build_id, result, runs_before) if result == 0: diff --git a/fluid_build/build_runners/debezium/iceberg_sink.py b/fluid_build/build_runners/debezium/iceberg_sink.py index 5d222378..54d45cc8 100644 --- a/fluid_build/build_runners/debezium/iceberg_sink.py +++ b/fluid_build/build_runners/debezium/iceberg_sink.py @@ -92,8 +92,11 @@ def emit_debezium_iceberg_sink_config( # GLOBAL (applies to all derived tables), not a per-table default. cfg["partition-by"] = ",".join(resolved.partition_by) - # Native Iceberg props (s3.* / gcs.* / adls.* / jdbc.* ...) pass straight - # through with the same prefix the runner adds. + # Native Iceberg props (s3.* / gcs.* / adls.* / jdbc.* / gcp.bigquery.* ...) + # pass straight through with the same prefix the runner adds: the server + # maps every ``debezium.sink.iceberg.*`` key into the catalog properties + # (memiiso/debezium-server-iceberg 1.2.0.Final IcebergConfig.java:18-21, + # IcebergChangeConsumer.java:100-102). for k, v in (resolved.extra_catalog_props or {}).items(): cfg[str(k)] = str(v) diff --git a/fluid_build/build_runners/debezium/runner.py b/fluid_build/build_runners/debezium/runner.py index 295f3fd1..bdfbb427 100644 --- a/fluid_build/build_runners/debezium/runner.py +++ b/fluid_build/build_runners/debezium/runner.py @@ -547,16 +547,19 @@ def _execute_debezium_server( if preflight_error: return _failed(ctx, started_at, t_start, preflight_error) if sink_plan is not None and sink_plan.derives: - from ...providers._iceberg_catalog import ( - find_iceberg_expose_binding, - resolve_iceberg_catalog, - ) + from ...providers._iceberg_catalog import resolve_iceberg_catalog + from ..kafka_connect.iceberg_sink_validation import iceberg_sink_target from .iceberg_sink import emit_debezium_iceberg_sink_config - binding = find_iceberg_expose_binding(ctx.contract) - if binding is not None: + # The exposes the build's outputs name, as `fluid validate` resolves + # them. They share one database and one catalog, so the first one's + # binding gives the namespace and the catalog block (for Glue, whose + # warehouse is a per-table prefix, the first one's prefix; a DynamoDB / + # JDBC warehouse is one value for all of them, or the resolver refuses). + targets, _error = iceberg_sink_target(ctx.contract, build, sink_plan) + if targets: resolved = resolve_iceberg_catalog( - binding, + targets[0].get("binding") or {}, contract=ctx.contract, sink=ctx.sink, account_ref=ctx.env.get("AWS_ACCOUNT_ID", ""), diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink.py b/fluid_build/build_runners/kafka_connect/iceberg_sink.py index 2066eb79..ef2e842d 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink.py @@ -111,6 +111,12 @@ def emit_iceberg_sink_config( cfg["iceberg.catalog.client.region"] = resolved.region if resolved.uri: cfg["iceberg.catalog.uri"] = resolved.uri + # Catalog-specific properties (``gcp.bigquery.project-id``...). The sink + # hands every ``iceberg.catalog.``-prefixed key to the catalog with the + # prefix stripped (apache-iceberg-1.10.0 kafka-connect/kafka-connect/src/ + # main/java/org/apache/iceberg/connect/IcebergSinkConfig.java:60, :254). + for k, v in (resolved.extra_catalog_props or {}).items(): + cfg[f"iceberg.catalog.{k}"] = str(v) # ── routing / write semantics ── if resolved.id_columns: diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 4aff656f..91455200 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -43,8 +43,13 @@ * ``sink.catalog`` must agree with the expose's catalog, because dbt and the IaC read only the expose, and so must the catalog an override or a hand-written config selects (its ``type`` or ``catalog-impl``); -* every ``binding.location`` key in the row's ``sink_requires`` must be set - (Glue keeps its advisory region warning: the warehouse falls back); +* every ``binding.location`` key in the row's ``sink_requires`` must be set, + a warehouse counting when the deriver resolves one (DynamoDB and JDBC derive + it from an explicit bucket); Glue keeps its advisory region warning: the + warehouse falls back; +* a bucket ``{{ env.* }}`` template the DynamoDB, JDBC or BigQuery warehouse + derives from, naming a variable that is unset or empty, is a warning here + and an error in the runner's preflight; * the runtime must ship the catalog's client (the stock Apache Iceberg Kafka Connect runtime has no Nessie client, and the published sink predates the ``bigquery`` catalog type); @@ -70,11 +75,26 @@ from __future__ import annotations +import contextlib +import contextvars import logging from dataclasses import dataclass -from typing import AbstractSet, Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from typing import ( + AbstractSet, + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, +) from ...providers._iceberg_catalog import ( + BIGQUERY_PROJECT_ID, + BUCKET_WAREHOUSE_KINDS, DYNAMODB_CATALOG_IMPL, FAMILY_GLUE, FAMILY_REST, @@ -88,6 +108,7 @@ iceberg_sink_exposes, known_catalog_kinds, resolve_iceberg_catalog, + resolve_iceberg_sink_exposes, ) LOG = logging.getLogger("fluid.acquire.iceberg_sink") @@ -151,6 +172,10 @@ class IcebergSinkPlan: overrides: Tuple[Tuple[str, Mapping[str, Any]], ...] #: ``(label, value)`` of the warehouse override the zero-drift check reads. warehouse_override: Tuple[str, Any] + #: The key that names the tables the sink writes (``iceberg.tables`` for + #: Kafka Connect, ``table-namespace`` for Debezium Server). An override map + #: that sets it names the tables itself instead of the derived config. + table_key: str = "" def iceberg_sink_plan(build: Mapping[str, Any]) -> Optional[IcebergSinkPlan]: @@ -191,6 +216,7 @@ def iceberg_sink_plan(build: Mapping[str, Any]) -> Optional[IcebergSinkPlan]: impl_key="catalog-impl", overrides=((label, config),), warehouse_override=(label, config.get("warehouse")), + table_key="table-namespace", ) if not declares_iceberg: @@ -217,9 +243,43 @@ def iceberg_sink_plan(build: Mapping[str, Any]) -> Optional[IcebergSinkPlan]: "iceberg_catalog_overrides", catalog_overrides.get("iceberg.catalog.warehouse"), ), + table_key="iceberg.tables", ) +def derives_table_identity(plan: IcebergSinkPlan) -> bool: + """Does forge-cli derive the tables ``plan``'s sink config writes? + + ``True`` when the runner derives the config and no override map sets + :attr:`IcebergSinkPlan.table_key`. Such a config writes the exposes the + build's outputs pick (:func:`iceberg_sink_target`). + """ + return plan.derives and not any(plan.table_key in m for _label, m in plan.overrides) + + +def iceberg_sink_target( + contract: Mapping[str, Any], + build: Mapping[str, Any], + plan: Optional[IcebergSinkPlan] = None, +) -> Tuple[Tuple[Mapping[str, Any], ...], Optional[str]]: + """The Iceberg sink exposes ``build``'s sink writes, or why there are none. + + THE build->expose join: ``fluid validate``, the runner preflight, both + streaming runners and the Kafka-Connect late-arrival target read it, so + they name the same tables. A config forge-cli derives + (:func:`derives_table_identity`) writes the exposes + :func:`resolve_iceberg_sink_exposes` joins from the build's outputs, by + the runtime's rule (one table for Kafka Connect, one namespace for + Debezium Server). Any other config names its own tables and is checked + against the first Iceberg sink expose. ``plan`` defaults to + :func:`iceberg_sink_plan` of ``build``. + """ + plan = plan if plan is not None else iceberg_sink_plan(build) + if plan is not None and derives_table_identity(plan): + return resolve_iceberg_sink_exposes(contract, build, namespace=plan.engine == "debezium") + return iceberg_sink_exposes(contract)[:1], None + + def validate_iceberg_sink(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: """Return (errors, warnings) for every Iceberg streaming-sink build.""" return _validate(contract) @@ -240,17 +300,29 @@ def iceberg_sink_preflight( a different catalog than dbt reads. ``build_id`` is the run context's id; the build is :func:`executing_build`, the one the runner reads its properties from, not one matched by message text. Warnings are logged and - never fatal, the same split as ``fluid validate`` without ``--strict``. + never fatal, the same split as ``fluid validate`` without ``--strict``, with + one difference: a ``location.bucket`` template that ``fluid validate`` + only warns about (it may resolve where the sink runs) is an error here, + because the runner derives the sink config in this process next. The + template is read from the contract as written when the runner runs inside + :func:`contract_as_written`, as the acquisition dispatcher runs it. """ build = executing_build(contract, build_id) if build is None: return None - errors, warnings = _validate(contract, builds=(build,)) + run_only: List[str] = [] + token = _AT_RUN.set(run_only) + try: + errors, warnings = _validate(contract, builds=(build,)) + finally: + _AT_RUN.reset(token) for msg in warnings: (log or LOG).warning("iceberg_sink.preflight.warning build=%s %s", build_id, msg) if not errors: return None - return "iceberg sink preflight failed (see `fluid validate`): " + "; ".join(errors) + # ``fluid validate`` reports every error but the run-only ones. + see = " (see `fluid validate`)" if any(e not in run_only for e in errors) else "" + return f"iceberg sink preflight failed{see}: " + "; ".join(errors) def _validate(contract: Mapping[str, Any], *, builds: Optional[Iterable[Any]] = None) -> _Issues: @@ -267,12 +339,13 @@ def _validate(contract: Mapping[str, Any], *, builds: Optional[Iterable[Any]] = runtime = iceberg_sink_plan(build) if runtime is None: continue # not a build that writes an Iceberg sink - _check_build(build, runtime, iceberg_exposes, expose_ids, errors, warnings) + _check_build(contract, build, runtime, iceberg_exposes, expose_ids, errors, warnings) return errors, warnings def _check_build( + contract: Mapping[str, Any], build: Mapping[str, Any], runtime: IcebergSinkPlan, iceberg_exposes: Sequence[Mapping[str, Any]], @@ -294,13 +367,19 @@ def _check_build( "the connector has no table identity to write to" ) return + # The exposes the runner writes, by the resolver the runners call. A + # derived config writes the exposes the build's outputs pick, and outputs + # that pick none it can write are an error; a config that names its own + # tables is checked against the first Iceberg expose. HARD. + targets, join_error = iceberg_sink_target(contract, build, runtime) + if join_error: + errors.append(join_error) outputs = build.get("outputs") or [] - if outputs and not (set(outputs) & expose_ids): + if not derives_table_identity(runtime) and outputs and not (set(outputs) & expose_ids): warnings.append( f"iceberg sink (build {bid!r}) outputs {list(outputs)} don't reference the " f"Iceberg expose(s) {sorted(x for x in expose_ids if x)}; the join is implicit" ) - binding = iceberg_exposes[0].get("binding") or {} # 2. upsert is deferred to v2 (locked v1 decision) — gate, don't silently # append-only. HARD. @@ -318,7 +397,22 @@ def _check_build( "streamingSink.routeField" ) - _check_catalog(bid, binding, sink, runtime, errors, warnings) + # The catalog checks read each expose the sink writes. A Debezium Server + # sink writes several only when they resolve to one catalog, so a message + # an earlier expose produced is not repeated. + auto_creates = _auto_creates(streaming, runtime) + for index, target in enumerate(targets): + binding = target.get("binding") or {} + if index == 0: + _check_catalog(bid, binding, sink, runtime, errors, warnings, auto_creates=auto_creates) + continue + more_errors: List[str] = [] + more_warnings: List[str] = [] + _check_catalog( + bid, binding, sink, runtime, more_errors, more_warnings, auto_creates=auto_creates + ) + errors.extend(m for m in more_errors if m not in errors) + warnings.extend(m for m in more_warnings if m not in warnings) def _check_catalog( @@ -328,8 +422,31 @@ def _check_catalog( runtime: IcebergSinkPlan, errors: List[str], warnings: List[str], + *, + auto_creates: bool = False, ) -> None: """The catalog checks (4-6), every one read off the kind's table row.""" + # The table the derived config names: ``.
`` for Kafka + # Connect's iceberg.tables, the namespace alone for Debezium Server's + # table-namespace. Without one the derived config named the table + # ``None.
`` (or Debezium Server's default namespace). HARD. + if derives_table_identity(runtime): + from ...providers._iceberg_catalog import iceberg_namespace + + location = binding.get("location") or {} + if not iceberg_namespace(binding): + errors.append( + f"iceberg sink (build {bid!r}): the expose's binding names no table namespace, " + "so the derived sink config has no namespace to write under. Set " + "binding.location.database (on GCP, binding.location.dataset)" + ) + elif runtime.engine == "kafka-connect" and not ( + location.get("table") or binding.get("table") + ): + errors.append( + f"iceberg sink (build {bid!r}): the expose's binding names no table, so the " + "derived iceberg.tables has none. Set binding.location.table" + ) loc = binding.get("location") or {} kind = iceberg_catalog_kind(binding, sink) info = catalog_kind_info(kind) @@ -382,16 +499,52 @@ def _check_catalog( ) # 4. catalog tagged-union completeness, from the row's ``sink_requires``. - # HARD: forge can't derive a REST uri or a catalog name. Glue requires - # nothing (the warehouse falls back), so its region stays advisory. + # HARD: forge can't derive a REST uri or a catalog name. A warehouse is + # present when the deriver resolves one (DynamoDB and JDBC derive it + # from an explicit bucket). A key the catalog needs only to start (the + # DynamoDB / JDBC warehouse, the BigQuery project) is also present when + # an override map sets its catalog property: the runner merges that map + # into the config it pushes. A bucket whose ``{{ env.* }}`` template + # does not resolve here is explicit, so ``fluid validate`` only warns: + # the runner's preflight, in the environment the sink config is + # derived in, refuses it when the sink would use that warehouse: a + # DynamoDB / JDBC catalog needs one to start, Debezium Server needs one + # to boot, and a Kafka Connect BigQuery sink reads it only to + # auto-create tables (see the BigQuery warning below). Otherwise the run + # warns too. Glue requires nothing (the warehouse falls back), so its + # region stays advisory. + from ...providers._iceberg_catalog import unset_bucket_env_vars + + derived_warehouse = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse + # Only a deriving build reads the bucket; one that does not pushes the + # operator's map as written. + as_written = _binding_as_written(binding) + waits_on: Tuple[str, ...] = () + if runtime.derives and not _overrides_warehouse(runtime): + waits_on = unset_bucket_env_vars(as_written, kind) + if waits_on: + required = "warehouse" in info.sink_requires + refused = required or runtime.engine == "debezium" or auto_creates + issue = _unresolved_bucket_issue( + bid, kind, as_written, waits_on, required=required, refused=refused + ) + run_only = _AT_RUN.get() + if run_only is None or not refused: + warnings.append(issue) + else: + errors.append(issue) + run_only.append(issue) for key in info.sink_requires: - if not loc.get(key): - suffix = ( - " (the catalog name)" if key == "warehouse" and info.family == FAMILY_REST else "" - ) + if key == "warehouse" and waits_on: + continue # reported above, naming the variables + present = derived_warehouse if key == "warehouse" else str(loc.get(key) or "").strip() + start_prop = _start_only_property(key, kind) + if not present and start_prop is not None: + present = _overrides_catalog_property(runtime, start_prop) + if not present: errors.append( f"iceberg sink (build {bid!r}): {kind} catalog requires " - f"binding.location.{key}{suffix}" + f"binding.location.{key}{_requires_hint(key, kind, info)}" ) if info.family == FAMILY_GLUE and not loc.get("region"): warnings.append( @@ -411,22 +564,249 @@ def _check_catalog( "the sink cannot load NessieCatalog" ) # ``type=bigquery`` (BigQueryMetastoreCatalog) is in Iceberg's CatalogUtil - # only from 1.10; the sink published on Confluent Hub is older, so a stock - # worker fails to load the catalog at connector start. Advisory: a runtime - # built from Iceberg >= 1.10 has it. + # only from 1.10 (apache-iceberg-1.10.0 CatalogUtil.java:78, :321-322; absent + # at apache-iceberg-1.9.2); the sink published on Confluent Hub is older, + # so a stock worker fails to load the catalog at connector start. + # Advisory: a runtime built from Iceberg >= 1.10 has it. if reaching == "bigquery" and runtime.engine == "kafka-connect": warnings.append( f"iceberg sink (build {bid!r}): the sink config sets " "iceberg.catalog.type=bigquery, which the published Apache Iceberg Kafka Connect " f"sink ({PUBLISHED_KAFKA_CONNECT_SINK_VERSION} on Confluent Hub) cannot load: " - "Iceberg's CatalogUtil gains the bigquery type in 1.10. Run a sink built from " - "Iceberg >= 1.10, or the connector fails at start" + "Iceberg's CatalogUtil gains the bigquery type in 1.10, so on that sink the " + "connector fails at start. Run a sink built from Iceberg >= 1.10" + ) + # BigQueryMetastoreCatalog reads the warehouse to place a dataset it + # creates, and a table it creates in a dataset with no default storage + # location URI; without one, both throw IllegalArgumentException + # (apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:146-161, :197-208, + # :298-303). The Kafka Connect sink's auto-create calls createNamespace + # before every createTable and catches only AlreadyExists / Forbidden + # (IcebergWriterFactory.java:89, :122-137), so every table auto-create + # throws, even in a dataset that exists. Debezium Server needs one to boot: + # IcebergConfig declares ``debezium.sink.iceberg.warehouse`` as a String + # with no default (memiiso/debezium-server-iceberg 1.2.0.Final + # IcebergConfig.java:44-45), in the config mapping GlobalConfig nests + # (GlobalConfig.java:14-15) and IcebergChangeConsumer injects + # (IcebergChangeConsumer.java:79), and SmallRye Config refuses to start + # with a required mapping property missing. Advisory; a bucket that waits + # on a variable is reported above. + if ( + reaching == "bigquery" + and kind == "bigquery" + and runtime.derives + and not derived_warehouse + and not waits_on + and not _overrides_warehouse(runtime) + ): + warnings.append( + f"iceberg sink (build {bid!r}): no gs:// warehouse can be derived from " + "binding.location (a gs:// warehouse, or a bucket), so the derived bigquery " + "catalog config sets none. " + + ( + "Debezium Server declares debezium.sink.iceberg.warehouse with no default " + "(debezium-server-iceberg 1.2.0.Final IcebergConfig), so the server refuses " + "to boot without it, whether or not the tables exist. Set " + "binding.location.warehouse or binding.location.bucket, or warehouse in " + "debezium.server.sink.config" + if runtime.engine == "debezium" + else "With iceberg.tables.auto-create-enabled the sink calls createNamespace " + "for every table it creates, and BigQueryMetastoreCatalog.createNamespace " + "refuses without a warehouse, so table auto-creation fails even in a dataset " + "that exists; without auto-create, tables that exist need none" + ) ) _check_warehouse_override(bid, binding, sink, kind, info, runtime, warnings) _check_selector_overrides(bid, kind, info, runtime, errors) +#: The run-only errors of the :func:`iceberg_sink_preflight` that is running, +#: ``None`` under ``fluid validate``. The runner derives the sink config in +#: the same process right after the preflight, so a ``location.bucket`` +#: template this environment cannot resolve leaves the sink with no +#: warehouse, or one in a bucket the contract does not name. +_AT_RUN: contextvars.ContextVar[Optional[List[str]]] = contextvars.ContextVar( + "iceberg_sink_preflight", default=None +) + +#: The contract as written, before the acquisition dispatcher resolved its +#: ``{{ env.* }}`` templates (see :func:`contract_as_written`). +_AS_WRITTEN: contextvars.ContextVar[Optional[Mapping[str, Any]]] = contextvars.ContextVar( + "iceberg_sink_contract_as_written", default=None +) + + +@contextlib.contextmanager +def contract_as_written(contract: Mapping[str, Any]) -> Iterator[None]: + """Let :func:`iceberg_sink_preflight` read ``contract`` as written. + + ``base._execute_acquisition_build`` resolves every ``{{ env.X }}`` before + the runner sees the contract, and a missing or empty ``X`` becomes ``""``: + ``acme-{{ env.LAKE_ENV }}-lake`` reaches the runner as ``acme--lake``. + Inside this block the preflight reads the bucket template from + ``contract``, so it names the variable and refuses the build instead of + passing a warehouse in a bucket the contract does not name. + """ + token = _AS_WRITTEN.set(contract) + try: + yield + finally: + _AS_WRITTEN.reset(token) + + +def _binding_as_written(binding: Mapping[str, Any]) -> Mapping[str, Any]: + """``binding`` as the contract wrote it, while a runner runs; else ``binding``. + + The expose binding in the contract :func:`contract_as_written` holds whose + resolution by the dispatcher's own resolver is ``binding``. Compared with + strings stripped: a plan-embedded contract is first resolved by + ``_contract_loader.resolve_contract_env_templates``, which strips every + string it renders. + """ + contract = _AS_WRITTEN.get() + if contract is None or _AT_RUN.get() is None: + return binding + return _match_as_written(contract, binding) + + +def unresolved_bucket_vars_at_run( + binding: Mapping[str, Any], kind: str, runtime: IcebergSinkPlan +) -> Tuple[str, ...]: + """The unset or empty variables the derived warehouse waits on, as check 4 reads them. + + For a runner deriving inside :func:`contract_as_written`, where ``binding`` + is already resolved: the variables in its bucket as the contract wrote it. + ``()`` outside that scope, for a build that does not derive, and when an + override map sets the warehouse (the override is what reaches the sink). + """ + contract = _AS_WRITTEN.get() + if contract is None or not runtime.derives or _overrides_warehouse(runtime): + return () + from ...providers._iceberg_catalog import unset_bucket_env_vars + + return unset_bucket_env_vars(_match_as_written(contract, binding), kind) + + +def _match_as_written(contract: Mapping[str, Any], binding: Mapping[str, Any]) -> Mapping[str, Any]: + """The expose binding in ``contract`` that resolves to ``binding``, else ``binding``.""" + from ..base import _resolve_env_placeholders + + target = _stripped(binding) + for expose in contract.get("exposes") or []: + raw = expose.get("binding") if isinstance(expose, Mapping) else None + if isinstance(raw, Mapping) and _stripped(_resolve_env_placeholders(dict(raw))) == target: + return raw + return binding + + +def _stripped(node: Any) -> Any: + """``node`` with every string leaf stripped of surrounding whitespace.""" + if isinstance(node, str): + return node.strip() + if isinstance(node, Mapping): + return {k: _stripped(v) for k, v in node.items()} + if isinstance(node, (list, tuple)): + return [_stripped(v) for v in node] + return node + + +def _unresolved_bucket_issue( + bid: Any, + kind: str, + binding: Mapping[str, Any], + names: Tuple[str, ...], + *, + required: bool, + refused: bool, +) -> str: + """Check 4's message for a warehouse that waits on unset or empty bucket variables.""" + raw = (binding.get("location") or {}).get("bucket") + one = len(names) == 1 + unset = f"{', '.join(names)} {'is' if one else 'are'} unset or empty" + refuses = f"the {kind} catalog refuses to start without a warehouse" + if _AT_RUN.get() is not None: + return ( + f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " + f"binding.location.bucket {raw!r}, and {unset} in the runner's environment, so " + "the bucket does not resolve: the sink would get a warehouse in a bucket the " + f"contract does not name, or none{f', and {refuses}' if required else ''}. " + f"Set {'it' if one else 'them'} here, or set binding.location.warehouse" + ) + return ( + f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " + f"binding.location.bucket {raw!r}, and {unset} here, so it cannot be derived at " + f"validate time. Set {'it' if one else 'them'} where the sink runs" + + ( + ": the runner refuses the build when the bucket does not resolve there" + + (f", because {refuses}" if required else "") + if refused + else "" + ) + ) + + +def _auto_creates(streaming: Mapping[str, Any], runtime: IcebergSinkPlan) -> bool: + """Does this Kafka Connect sink auto-create tables? Its default is off. + + ``streamingSink.autoCreate`` sets ``iceberg.tables.auto-create-enabled``, + and an override map merged after it wins (apache-iceberg-1.10.0 + IcebergSinkConfig.java:154-159 defaults the key to false). + """ + value: Any = streaming.get("autoCreate") + for _label, cfg in runtime.overrides: + if "iceberg.tables.auto-create-enabled" in cfg: + value = cfg["iceberg.tables.auto-create-enabled"] + return str(value).strip().lower() == "true" + + +def _requires_hint(key: str, kind: str, info: CatalogKind) -> str: + """What a missing ``sink_requires`` key is to the sink, for check 4's error.""" + if key == "warehouse" and info.family == FAMILY_REST: + return " (the catalog name)" + if key == "warehouse" and kind in BUCKET_WAREHOUSE_KINDS: + return ( + " (an object-store location), or a binding.location.bucket on platform aws or " + "gcp to derive it from, or the sink's warehouse property in an override; the " + f"{kind} catalog refuses to start without a warehouse" + ) + if key == "project" and kind == "bigquery": + return ( + f" (the sink's {BIGQUERY_PROJECT_ID}), or that property in an override; the " + "bigquery catalog refuses to start without it" + ) + return "" + + +def _start_only_property(key: str, kind: str) -> Optional[str]: + """The catalog property behind a ``sink_requires`` key only the sink needs. + + DynamoDbCatalog and JdbcCatalog refuse to start without ``warehouse``, and + BigQueryMetastoreCatalog without ``gcp.bigquery.project-id`` (the rows in + ``_iceberg_catalog`` cite the upstream lines), so an override map that sets + the property supplies what the sink needs. ``None`` for any other key. + """ + if key == "warehouse" and kind in BUCKET_WAREHOUSE_KINDS: + return "warehouse" + if key == "project" and kind == "bigquery": + return BIGQUERY_PROJECT_ID + return None + + +def _overrides_catalog_property(runtime: IcebergSinkPlan, prop: str) -> bool: + """Does an override map merged over the derived config set catalog property ``prop``?""" + # A catalog property carries the same runtime prefix as the type key: + # ``iceberg.catalog.warehouse`` on Kafka Connect, bare ``warehouse`` on + # Debezium Server. + key = runtime.type_key[: -len("type")] + prop + return any(mapping.get(key) for _label, mapping in runtime.overrides) + + +def _overrides_warehouse(runtime: IcebergSinkPlan) -> bool: + """Does an override map merged over the derived config set the warehouse?""" + return _overrides_catalog_property(runtime, "warehouse") + + def _reaching_catalog_type(info: Any, runtime: IcebergSinkPlan) -> Optional[str]: """The catalog ``type`` the worker actually receives, or ``None``. @@ -573,7 +953,9 @@ def _check_warehouse_override( derived_wh = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse if not derived_wh and "warehouse" in info.sink_requires: - return # check 4 already refused the missing warehouse + # The binding has no warehouse to diverge from: check 4 refused it, or + # (DynamoDB / JDBC) accepted the override as the sink's warehouse. + return if same_warehouse(override_wh, derived_wh): return if info.family == FAMILY_GLUE: diff --git a/fluid_build/build_runners/kafka_connect/runner.py b/fluid_build/build_runners/kafka_connect/runner.py index 29067489..7ed93e08 100644 --- a/fluid_build/build_runners/kafka_connect/runner.py +++ b/fluid_build/build_runners/kafka_connect/runner.py @@ -95,15 +95,20 @@ def resolve_sink_connector(format_or_platform: str, override: Optional[str] = No return SINK_CONNECTOR_CLASS["s3"] # safe default -def _find_iceberg_expose_binding(contract: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """The expose ``binding`` carrying the Iceberg-table identity for a sink. - - Delegates to the shared resolver helper so the Kafka-Connect and - Debezium-Server runners pick the SAME table (RFC zero-drift spine). +def _find_iceberg_expose_binding( + contract: Mapping[str, Any], build: Mapping[str, Any] +) -> Optional[Dict[str, Any]]: + """The expose ``binding`` carrying the Iceberg-table identity for ``build``'s sink. + + Delegates to the shared resolver so the Kafka-Connect and Debezium-Server + runners pick the SAME table (RFC zero-drift spine): the expose + ``iceberg_sink_target`` resolves, the one ``fluid validate`` checks. + ``None`` when it resolves none (the preflight refuses that build first). """ - from ...providers._iceberg_catalog import find_iceberg_expose_binding + from .iceberg_sink_validation import iceberg_sink_target - return find_iceberg_expose_binding(contract) + exposes, _error = iceberg_sink_target(contract, build) + return dict(exposes[0].get("binding") or {}) if exposes else None # ── REST client ──────────────────────────────────────────────────────── @@ -338,10 +343,13 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: # only; nothing provisions the side table yet (RFC §6.7 / spike §14). late_arrival_target = connector_name_for_topic if str(ctx.sink.format or "").lower() == "iceberg": - _ib = _find_iceberg_expose_binding(ctx.contract) or {} + _ib = _find_iceberg_expose_binding(ctx.contract, build) or {} + from ...providers._iceberg_catalog import iceberg_namespace + _loc = _ib.get("location") or {} - if _loc.get("database") and _loc.get("table"): - late_arrival_target = f"{_loc['database']}.{_loc['table']}" + _namespace = iceberg_namespace(_ib) + if _namespace and _loc.get("table"): + late_arrival_target = f"{_namespace}.{_loc['table']}" late_arrival_policy = extract_late_arrival_policy( contract_or_source=ctx.source, @@ -370,7 +378,7 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: if preflight_error: return _failed(ctx, started_at, t_start, preflight_error) if sink_plan is not None and sink_plan.derives: - binding = _find_iceberg_expose_binding(ctx.contract) + binding = _find_iceberg_expose_binding(ctx.contract, build) if binding is not None: from ...providers._iceberg_catalog import resolve_iceberg_catalog from .iceberg_sink import emit_iceberg_sink_config @@ -392,6 +400,14 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: schema_registry_url=sr_url, delivery_guarantee=(props.get("delivery") or {}).get("guarantee"), ) + # A bucket whose variable is unset or empty here resolved to one the + # contract does not name. The preflight lets that through only for a + # sink that never reads the warehouse (BigQuery without auto-create); + # an override warehouse is the operator's and stays. + from .iceberg_sink_validation import unresolved_bucket_vars_at_run + + if unresolved_bucket_vars_at_run(binding, resolved.kind, sink_plan): + derived.pop("iceberg.catalog.warehouse", None) sink_config = {**derived, **(sink_config or {})} connector_name = kc_props.get("connector_name") or f"forge-{ctx.product_id.replace('.', '-')}" diff --git a/fluid_build/cli/policy_apply.py b/fluid_build/cli/policy_apply.py index 2b4587b4..97d21174 100644 --- a/fluid_build/cli/policy_apply.py +++ b/fluid_build/cli/policy_apply.py @@ -22,7 +22,7 @@ from ..observability.tracing import traced_stage as _traced_stage from ._common import CLIError, build_provider -from ._logging import info +from ._logging import info, warn from .console import cprint COMMAND = "policy-apply" @@ -87,6 +87,17 @@ def run(args, logger: logging.Logger) -> int: with open(bindings_path, encoding="utf-8") as f: data = json.load(f) + # The compiler's warnings, echoed here too: a grant that compiled to + # no binding (see ``fluid policy compile``) is not enforced by this + # step either, and apply can run in a different job from compile. A + # contract with no grants leaves nothing unenforced. + from fluid_build.policy.compiler import NO_GRANTS + + stored = data.get("warnings") if isinstance(data, dict) else None + for warning in stored if isinstance(stored, list) else (): + if warning != NO_GRANTS: + warn(logger, "policy_bindings_warning", warning=warning, bindings=args.bindings) + # An empty bindings file is a legitimate no-op, not an error: a # contract with no ``accessPolicy`` grants — every raw bronze / SDP # acquisition product — compiles to zero bindings. There is nothing diff --git a/fluid_build/cli/policy_compile.py b/fluid_build/cli/policy_compile.py index 13c805db..df2823c7 100644 --- a/fluid_build/cli/policy_compile.py +++ b/fluid_build/cli/policy_compile.py @@ -17,9 +17,11 @@ import argparse import logging import os +import re +from typing import Any, List from ._common import CLIError, load_contract_with_overlay, write_json -from ._logging import info +from ._logging import info, warn COMMAND = "policy-compile" @@ -47,23 +49,82 @@ def register(subparsers: argparse._SubParsersAction): _add_arguments(p) +#: The values ``compile_policy`` reads as a mapping or a list. A wrong type at +#: one of these paths is one the compiler can fail on. Any other schema +#: error (an extra key on a grant, a wrong type under ``metadata``) is not +#: named: it did not make the compiler fail. +_COMPILER_READ_PATHS = re.compile( + r"accessPolicy(?:\.grants(?:\[\d+\](?:\.permissions)?)?)?" + r"|exposes\[\d+\]\.binding(?:\.location)?" +) + + +def _compiler_schema_errors(contract: Any) -> List[str]: + """The schema type errors where the compiler reads, as `` is not of type ``. + + ``policy compile`` does not schema-validate, so a grant or binding of the + wrong type reaches the compiler and fails there as a Python error that + names neither the value nor the fix. Read only after the compiler failed; + ``[]`` when the schema finds no such error or cannot be run. + """ + try: + from fluid_build.schema_manager import FluidSchemaManager + + result = FluidSchemaManager().validate_contract(contract, offline_only=True) + except Exception: # the compiler's own error is reported instead + return [] + # An error starts with its path: ``accessPolicy.grants[0]: ...``, + # ``exposes[0].binding: ...``. + found = [] + for error in result.errors: + path, _, message = error.partition(": ") + if _COMPILER_READ_PATHS.fullmatch(path) and " is not of type " in message: + # The path and the expected type, never the value: a value of the + # wrong type can be a connection URL with a password in it. + expected = message.rsplit(" is not of type ", 1)[1] + found.append(f"{path} is not of type {expected}") + return found + + +def _compile_failed(contract: Any, exc: Exception) -> CLIError: + """The ``policy_compiler_crashed`` error for a crash inside ``compile_policy``.""" + schema_errors = _compiler_schema_errors(contract) + if schema_errors: + error = "policy compile failed on contract values of the wrong type: " + "; ".join( + schema_errors + ) + else: + error = f"policy compiler failed: {type(exc).__name__}: {exc}" + return CLIError(1, "policy_compiler_crashed", {"error": error}) + + def run(args, logger: logging.Logger) -> int: import traceback try: c = load_contract_with_overlay(args.contract, getattr(args, "env", None), logger) try: - from fluid_build.policy.compiler import compile_policy + from fluid_build.policy.compiler import NO_GRANTS, compile_policy bindings, warnings = compile_policy(c) except Exception as e: - logger.error(f"Policy compiler error: {e}") - logger.error(traceback.format_exc()) - bindings, warnings = {"bindings": []}, [f"policy compiler failed: {str(e)}"] + # Fail closed. This used to write an empty bindings list and exit + # 0, so a crash read as "no grants to enforce" to the next stage. + # Nothing is written: a bindings file from an earlier run stays as + # it was, and the non-zero exit stops the pipeline before apply. + logger.debug(traceback.format_exc()) + raise _compile_failed(c, e) from e out_dir = os.path.dirname(args.out) if out_dir: # Only create dir if path has a directory component os.makedirs(out_dir, exist_ok=True) write_json(args.out, {"bindings": bindings, "warnings": warnings}) + # A warning can be the only sign that a grant compiled to nothing (an + # Iceberg table in a catalog the compiler cannot grant on), so it goes + # to the console, not only into the file. Warnings keep exit 0. A + # contract with no grants leaves nothing unenforced: not a WARNING. + for warning in warnings: + if warning != NO_GRANTS: + warn(logger, "policy_compile_warning", warning=warning, out=args.out) info(logger, "policy_compiled", out=args.out, warnings=len(warnings)) return 0 except CLIError: diff --git a/fluid_build/iac/catalog_moves.py b/fluid_build/iac/catalog_moves.py index 4514e511..9cfffcd7 100644 --- a/fluid_build/iac/catalog_moves.py +++ b/fluid_build/iac/catalog_moves.py @@ -128,7 +128,9 @@ class CatalogMoveSpec: ``addresses_before`` returns addresses the old release created for a moved binding that do not depend on its location, for a resource the before emit cannot re-derive once the location changed too; the default is none. - The prose fields fill the error message. + The prose fields fill the error message. With no ``evidence_types``, the + message does not attribute the resources to the moved exposes: + ``consequence`` says why they may be in state, and the exposes follow it. """ cloud: str @@ -138,7 +140,8 @@ class CatalogMoveSpec: as_before: Callable[[MutableMapping[str, Any]], None] #: Counted noun for the resources, e.g. "Glue catalog resource(s)". noun: str - #: Why the plan would destroy them, and what that destroy does. + #: Why the plan would destroy them, and what that destroy does. With no + #: ``evidence_types`` it also introduces the expose list printed after it. consequence: str #: Where ``state rm`` leaves them, e.g. "AWS". stays_in: str @@ -276,17 +279,20 @@ def _snowflake_volume_before( as_before=_back_into_snowflake, noun="Snowflake EXTERNAL VOLUME(s)", consequence=( - "forge-cli no longer creates an EXTERNAL VOLUME for an Iceberg table in a catalog " - "Snowflake does not manage (dbt now writes it as an externally cataloged table, " - "not a Snowflake-managed one on a volume), so applying now would plan to DROP " - "these volumes, and any Snowflake-managed Iceberg table an earlier dbt run wrote " - "onto one still uses it." + "The volume is named for the contract, not for an expose, so the state does not " + "say which expose it was created for. Possible causes: this contract was applied " + "by a forge-cli release that gave an Iceberg table in a catalog Snowflake does not " + "manage an EXTERNAL VOLUME (this release gives it none, and dbt writes it as an " + "externally cataloged table); or this change removed a Snowflake-managed Iceberg " + "expose, or moved one to another catalog. Applying now would plan to DROP the " + "volume, and any Snowflake-managed Iceberg table written onto it still uses it.\n\n" + "Iceberg exposes whose catalog earlier releases gave an EXTERNAL VOLUME:" ), stays_in="Snowflake", afterwards=( "Drop a volume by hand (DROP EXTERNAL VOLUME) only once no Iceberg table uses it. " - "If the table belongs in Snowflake's own catalog, remove location.catalog (or set " - "it to 'snowflake') instead." + "If an Iceberg table belongs in Snowflake's own catalog, remove its " + "location.catalog (or set it to 'snowflake') instead." ), addresses_before=_snowflake_volume_before, ) @@ -485,15 +491,24 @@ def guard_catalog_moves( if not addresses: return commands = _state_rm_commands(addresses, workdir=workdir) + expose_lines = "\n".join(f" exposes[{e}]: location.catalog {k}" for e, k in exposes) + address_lines = "\n".join(f" {a}" for a in addresses) + if spec.evidence_types: + found = ( + f"{len(addresses)} {spec.noun} for Iceberg table(s) that now live in another " + f"catalog:\n\n{expose_lines}\n\n{address_lines}\n\n{spec.consequence}" + ) + else: + # No resource in state ties these to an expose, so the moved exposes + # are one possible cause, listed after the prose that says so. + found = ( + f"{len(addresses)} {spec.noun} that this contract's configuration no longer " + f"declares:\n\n{address_lines}\n\n{spec.consequence}\n\n{expose_lines}" + ) raise CatalogMoveError( "iceberg catalog move blocked — this contract's OpenTofu state holds " - f"{len(addresses)} {spec.noun} for Iceberg table(s) that now live in another " - "catalog:\n\n" - + "\n".join(f" exposes[{e}]: location.catalog {k}" for e, k in exposes) - + "\n\n" - + "\n".join(f" {a}" for a in addresses) - + f"\n\n{spec.consequence}\n\n" - "Drop each from this contract's state, then re-run apply:\n\n" + + found + + "\n\nDrop each from this contract's state, then re-run apply:\n\n" + "\n".join(f" {command}" for command in commands) + "\n\n`tofu state rm` touches ZERO bytes of infrastructure: the resources stay in " f"{spec.stays_in}, and only this contract's claim on them is released. " + spec.afterwards, diff --git a/fluid_build/iac/providers/confluent.py b/fluid_build/iac/providers/confluent.py index 9fbd0e30..9627605a 100644 --- a/fluid_build/iac/providers/confluent.py +++ b/fluid_build/iac/providers/confluent.py @@ -40,7 +40,7 @@ from typing import Any, Dict, Iterable, List, Mapping, Tuple -from ...providers._iceberg_catalog import binding_catalog_kind, canonical_catalog_kind +from ...providers._iceberg_catalog import binding_catalog_kind from ...providers.aws.util.warehouse import normalize_location from ..importer import ImportBlock from ..naming import safe_ident, tofu_ref @@ -84,11 +84,10 @@ def _non_glue_catalog(binding: Mapping[str, Any]) -> str: less claim on the name, in a catalog the contract never named. The emitter and ``validate_confluent_binding`` both read this, so the gate refuses exactly what the emitter declines to publish. An absent - ``catalog`` is Glue here, as it always was. + ``catalog`` is Glue: ``_iceberg_catalog.default_catalog_kind`` gives Glue for + ``platform: confluent``, so the policy compiler and dbt read the same + catalog this emitter publishes to. """ - loc = binding.get("location") or {} - if not canonical_catalog_kind(loc.get("catalog")): - return "" kind = binding_catalog_kind(binding) return "" if kind == "glue" else kind diff --git a/fluid_build/policy/compiler.py b/fluid_build/policy/compiler.py index 1b6d64a0..89ddc51e 100644 --- a/fluid_build/policy/compiler.py +++ b/fluid_build/policy/compiler.py @@ -51,6 +51,12 @@ "manage": ["INSERT", "UPDATE", "DELETE", "SELECT"], } +#: The message for a contract with no grants. It stays in ``warnings`` (and +#: so in ``bindings.json``), but it reports a legitimate no-op, not a grant +#: left unenforced, so ``fluid policy compile`` / ``apply`` do not show it as +#: a WARNING. +NO_GRANTS = "No grants found in accessPolicy" + def compile_policy(contract: dict) -> Tuple[List[Dict[str, Any]], List[str]]: """Compile accessPolicy from contract into provider IAM bindings. @@ -70,7 +76,7 @@ def compile_policy(contract: dict) -> Tuple[List[Dict[str, Any]], List[str]]: grants = (contract.get("accessPolicy") or {}).get("grants", []) if not grants: - warnings.append("No grants found in accessPolicy") + warnings.append(NO_GRANTS) return bindings, warnings for g in grants: @@ -93,6 +99,10 @@ def compile_policy(contract: dict) -> Tuple[List[Dict[str, Any]], List[str]]: _compile_catalog_managed_iceberg( bindings, warnings, exp, binding, principal, permissions ) + elif _is_tableflow(binding): + _compile_tableflow_bindings( + bindings, warnings, exp, fmt, loc, principal, permissions + ) elif platform == "aws" or fmt in ("s3_file", "iceberg", "parquet"): _compile_aws_bindings(bindings, fmt, loc, principal, permissions) elif platform == "snowflake" or fmt == "snowflake_table": @@ -216,6 +226,49 @@ def _compile_catalog_managed_iceberg(bindings, warnings, exp, binding, principal ) +def _is_tableflow(binding) -> bool: + """Is this a Glue-cataloged Iceberg expose the Confluent Tableflow IaC publishes? + + Matched the way that emitter matches its exposes (``is_cloud``), so a + platform alias it publishes is compiled here too. + """ + from ..iac.provider_match import is_cloud + from ..providers._iceberg_catalog import is_iceberg_format + + return is_cloud(binding, "confluent") and is_iceberg_format(binding.get("format")) + + +def _compile_tableflow_bindings(bindings, warnings, exp, fmt, loc, principal, permissions): + """Compile a Confluent Tableflow expose: its bucket and the Glue table it publishes. + + Tableflow publishes the table for the Kafka topic, not ``location.table``: + the Tableflow IaC names it with ``_topic_name`` (topic > table > exposeId) + and publishes it into ``location.database`` (``custom_database``). With no + database, Tableflow's Glue database is the Kafka cluster id ("By default, + the Glue database name is the cluster ID, and each topic becomes a table + under it": docs.confluent.io/cloud/current/topics/tableflow/how-to-guides/ + catalog-integration/integrate-with-aws-glue-catalog). When neither names + the database, the Glue grant is reported as a warning, not widened to a + database. + """ + from ..iac.providers.confluent import _topic_name + + topic = _topic_name(loc, exp) + database = loc.get("database") or loc.get("kafka_cluster_id") + if database: + published = {**loc, "database": database, "dataset": None, "table": topic} + _compile_aws_bindings(bindings, fmt, published, principal, permissions) + return + _compile_aws_bindings(bindings, fmt, loc, principal, permissions, glue=False) + expose_id = exp.get("exposeId") or exp.get("id") or "?" + warnings.append( + f"Iceberg expose '{expose_id}' (platform confluent) names neither " + f"binding.location.database nor kafka_cluster_id, so the Glue database Tableflow " + f"publishes topic '{topic}' into is unknown and no Glue table grant was compiled " + f"for {principal} {list(permissions)}." + ) + + def _compile_aws_bindings(bindings, fmt, loc, principal, permissions, *, glue=True): """Generate AWS IAM policy statements. diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 956bf1b9..003923de 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -26,13 +26,19 @@ from __future__ import annotations import hashlib +import os import re from dataclasses import dataclass, field from types import MappingProxyType from typing import Any, Dict, Mapping, Optional, Tuple from ._sql_safety import validate_ident -from .aws.util.warehouse import get_iceberg_warehouse +from .aws.util.warehouse import ( + _ENV_TEMPLATE_RE, + get_iceberg_warehouse, + normalize_location, + resolve_env_templates, +) # Apache Iceberg runtime class names (pinned to the connector surface validated # in the OSS spike — RFC §14). Bumping the Iceberg runtime may change these. @@ -138,12 +144,23 @@ def speaks_rest(self) -> bool: # Connect runtime does not bundle iceberg-nessie. CatalogKind("nessie", FAMILY_NATIVE, "nessie", None, "iceberg_rest", _REST_REQUIRES), # BigLake metastore. ``type=bigquery`` exists in Iceberg >= 1.10. - CatalogKind("bigquery", FAMILY_NATIVE, "bigquery", None, "iceberg_rest"), + # BigQueryMetastoreCatalog.initialize refuses to start without + # ``gcp.bigquery.project-id``, which the resolver maps from + # ``location.project`` (apache-iceberg-1.10.0 + # BigQueryMetastoreCatalog.java:84-87). + CatalogKind("bigquery", FAMILY_NATIVE, "bigquery", None, "iceberg_rest", ("project",)), CatalogKind("hive", FAMILY_NATIVE, "hive", None, None), - CatalogKind("jdbc", FAMILY_NATIVE, "jdbc", None, None, ("uri",)), + # JdbcCatalog.initialize refuses a missing uri and an empty warehouse + # (apache-iceberg-1.10.0 JdbcCatalog.java:114-119). The warehouse + # may be derived: see :data:`BUCKET_WAREHOUSE_KINDS`. + CatalogKind("jdbc", FAMILY_NATIVE, "jdbc", None, None, ("uri", "warehouse")), CatalogKind("hadoop", FAMILY_NATIVE, "hadoop", None, None, ("warehouse",)), # CatalogUtil has no ``dynamodb`` type: it is reached by impl only. - CatalogKind("dynamodb", FAMILY_NATIVE, None, DYNAMODB_CATALOG_IMPL, None), + # DynamoDbCatalog.initialize refuses an empty warehouse + # (apache-iceberg-1.10.0 DynamoDbCatalog.java:132-134). + CatalogKind( + "dynamodb", FAMILY_NATIVE, None, DYNAMODB_CATALOG_IMPL, None, ("warehouse",) + ), # Horizon over an EXTERNAL VOLUME; a streaming writer reaches it # through Snowflake's Iceberg REST endpoint. CatalogKind( @@ -158,6 +175,22 @@ def speaks_rest(self) -> bool: } ) +#: Kinds whose ``warehouse`` is an object-store location the resolver may +#: derive from an explicit ``location.bucket`` (and ``path``) when +#: ``location.warehouse`` is absent. Never from the account-derived fallback +#: bucket the Glue row uses: no IaC creates that bucket for a non-Glue table. +BUCKET_WAREHOUSE_KINDS = frozenset({"dynamodb", "jdbc"}) + +#: The object-store scheme a derived warehouse gets, by canonical platform. +_BUCKET_WAREHOUSE_SCHEMES: Mapping[str, str] = MappingProxyType({"aws": "s3", "gcp": "gs"}) + +#: BigQueryMetastoreCatalog's own property names (apache-iceberg-1.10.0 +#: bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/ +#: BigQueryMetastoreCatalog.java:62-63). Without a location the catalog uses +#: ``us`` (:68, :90). +BIGQUERY_PROJECT_ID = "gcp.bigquery.project-id" +BIGQUERY_LOCATION = "gcp.bigquery.location" + #: Accepted spellings for a canonical kind (after case and ``-``/``_`` folding). _CATALOG_ALIASES: Mapping[str, str] = MappingProxyType( {"iceberg-rest": "rest", "snowflake": "snowflake-managed"} @@ -200,8 +233,12 @@ def known_catalog_kinds() -> Tuple[str, ...]: def default_catalog_kind(binding: Mapping[str, Any]) -> str: - """The kind an Iceberg binding gets when it names none: Glue on AWS, - Snowflake-managed on Snowflake, a REST catalog anywhere else. + """The kind an Iceberg binding gets when it names none: Glue on AWS and on + Confluent, Snowflake-managed on Snowflake, a REST catalog anywhere else. + + Confluent is Glue because the Tableflow IaC publishes a table with no + ``catalog`` to AWS Glue (``iac/providers/confluent.py``), so the policy + compiler and dbt read the catalog the table is actually in. GCP is the one platform where an absent catalog means two things: the streaming sink has always written a REST catalog there, while dbt-bigquery @@ -213,7 +250,7 @@ def default_catalog_kind(binding: Mapping[str, Any]) -> str: from ..iac.provider_match import canonical_cloud cloud = canonical_cloud(binding.get("platform")) - if cloud == "aws": + if cloud in ("aws", "confluent"): return "glue" if cloud == "snowflake": return "snowflake-managed" @@ -298,17 +335,231 @@ def iceberg_sink_exposes(contract: Mapping[str, Any]) -> Tuple[Mapping[str, Any] def find_iceberg_expose_binding(contract: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """The expose ``binding`` carrying the Iceberg-table identity for a sink. - - Shared by the Kafka-Connect and Debezium-Server runners so both resolve the - SAME table identity (the RFC zero-drift spine): the first of - :func:`iceberg_sink_exposes`. The validated build->expose join - (build.outputs / exposeId) lands with the plan-time validator (RFC §6.8 #5). + """The binding of the first of :func:`iceberg_sink_exposes`, or ``None``. + + A sink config that names its own tables (hand-written, or an override that + sets ``iceberg.tables`` / ``table-namespace``) is checked against this + expose's catalog. A sink config forge-cli derives reads the exposes + :func:`resolve_iceberg_sink_exposes` joins from the build's outputs; the + streaming runners select between the two through + ``iceberg_sink_validation.iceberg_sink_target``. """ exposes = iceberg_sink_exposes(contract) return dict(exposes[0].get("binding") or {}) if exposes else None +def _expose_id(expose: Mapping[str, Any]) -> Any: + return expose.get("exposeId") or expose.get("id") + + +def iceberg_namespace(binding: Mapping[str, Any]) -> str: + """The Iceberg namespace a sink writes ``binding``'s table under, or ``""``. + + ``location.database``, else a top-level ``database``, else + ``location.dataset``: a GCP binding names its BigQuery dataset there, and + the BigQuery metastore catalog maps a namespace to a dataset + (apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:149, :200, :307). + """ + loc = binding.get("location") or {} + value = loc.get("database") or binding.get("database") or loc.get("dataset") + return str(value or "").strip() + + +def _expose_database(expose: Mapping[str, Any]) -> Any: + """The database leg of an expose's table, read as :func:`resolve_iceberg_catalog` reads it.""" + return iceberg_namespace(expose.get("binding") or {}) or None + + +def _described(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: + """``["orders (sales.orders)", ...]``: each expose's id and table, for the messages.""" + return [ + f"{_expose_id(e)} ({resolve_iceberg_catalog(e.get('binding') or {}).fq_table})" + for e in exposes + ] + + +def resolve_iceberg_sink_exposes( + contract: Mapping[str, Any], + build: Mapping[str, Any], + *, + namespace: bool = False, +) -> Tuple[Tuple[Mapping[str, Any], ...], Optional[str]]: + """The exposes a sink config forge-cli DERIVES for ``build`` writes, or why it cannot. + + The candidates are :func:`iceberg_sink_exposes`; the build's ``outputs`` + pick among them, and with no outputs the candidates are the contract's. + + * ``namespace=False`` (Kafka Connect): the derived config carries one + ``iceberg.tables`` entry, and the Apache Iceberg sink writes each record + to every table that lists (``SinkWriter.routeRecordStatically``), so it + writes one expose. Exactly one candidate in outputs, or no outputs and + exactly one in the contract; anything else is an error. + * ``namespace=True`` (embedded Debezium Server): the derived config carries + one ``table-namespace`` and one catalog, and the server writes each + captured table under that namespace (``DefaultIcebergTableMapper``). + The candidates selected must share one database and resolve to one + catalog; outputs that name none of them are an error. + + Returns ``(exposes, None)`` with at least one expose, ``((), error)``, or + ``((), None)`` when the contract has no Iceberg sink expose at all (the + validator reports that). + """ + candidates = iceberg_sink_exposes(contract) + if not candidates: + return (), None + head = f"iceberg sink (build {build.get('id', '?')!r})" + raw_outputs = build.get("outputs") or [] + outputs = [raw_outputs] if isinstance(raw_outputs, str) else list(raw_outputs) + selected = tuple(e for e in candidates if _expose_id(e) in outputs) if outputs else candidates + if outputs: + picked = f"its outputs {outputs} name {len(selected) or 'none'} of the" + else: + picked = f"it declares no outputs, and the contract has {len(candidates)}" + picked += f" Iceberg sink exposes {_described(candidates)}" + if outputs: + writes = f"the exposes its outputs {outputs} name, {_described(selected)}," + else: + writes = f"the contract's Iceberg sink exposes, {_described(selected)}," + + if not namespace: + if len(selected) == 1: + return selected, None + return (), ( + f"{head}: {picked}; a derived Kafka Connect sink writes one expose (one " + "iceberg.tables entry). List exactly one of them in the build's outputs, split " + "the build into one build per expose, or hand-write the sink config " + "(properties.kafka-connect.sink_connector_config)" + ) + + remedy = "hand-write the sink config (properties.debezium.server.sink.config)" + if not selected: + return (), ( + f"{head}: {picked}; a derived Debezium Server sink writes the exposes the " + f"build's outputs name. List them in the build's outputs, or {remedy}" + ) + databases = sorted({str(_expose_database(e)) for e in selected}) + if len(databases) > 1: + return (), ( + f"{head}: a derived Debezium Server sink writes every captured table under one " + f"table-namespace, but {writes} sit in the databases {databases}. Give them one " + f"binding.location.database (location.dataset on GCP), split the build into one " + f"build per database, or {remedy}" + ) + differs = _catalog_differences(selected) + derived = _derived_warehouses(selected) if differs == ["warehouse"] else {} + if derived: + kind = resolve_iceberg_catalog(selected[0].get("binding") or {}).kind + return (), ( + f"{head}: a derived Debezium Server sink writes every table through one {kind} " + f"catalog warehouse, and the catalog creates a missing table under that " + f"warehouse ({_BUCKET_WAREHOUSE_LAYOUT[kind]}). But {writes} derive different " + f"warehouses from binding.location.bucket and path ({derived}): each is that " + f"expose's own table prefix, so one expose's tables would be created inside " + f"another expose's prefix. Set one binding.location.warehouse on them, split the " + f"build into one build per expose, or {remedy}" + ) + if differs: + return (), ( + f"{head}: a derived Debezium Server sink writes through one catalog, but {writes} " + f"resolve to different catalogs (their {', '.join(differs)} differ). Give them one " + f"catalog, split the build into one build per catalog, or {remedy}" + ) + return selected, None + + +#: The binding setting each :class:`ResolvedIcebergCatalog` catalog field is +#: resolved from, for the messages. +_CATALOG_FIELD_SOURCES: Mapping[str, str] = MappingProxyType( + { + "kind": "catalog", + "catalog_type": "catalog", + "catalog_impl": "catalog", + "uri": "uri", + "warehouse": "warehouse", + "io_impl": "warehouse", + "region": "region", + "extra_catalog_props": "catalog properties", + } +) + +#: The binding setting each catalog-specific property is resolved from. +_CATALOG_PROP_SOURCES: Mapping[str, str] = MappingProxyType( + {BIGQUERY_PROJECT_ID: "project", BIGQUERY_LOCATION: "region"} +) + + +def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: + """The catalog settings on which ``exposes`` resolve differently, or ``[]``. + + Compares every :class:`ResolvedIcebergCatalog` field except the table + identity and the per-table write defaults. A Glue warehouse is per table + too (``get_iceberg_warehouse`` derives ``s3://{bucket}/{database}/{table}/``), + while a REST catalog's warehouse is the catalog's name. A DynamoDB or JDBC + warehouse derived from ``location.bucket`` (:func:`_bucket_warehouse`) is + compared whole: the catalog creates every missing table under the one + warehouse the sink is given, so it is not a per-table prefix. + """ + from dataclasses import fields as dataclass_fields + + resolved = [resolve_iceberg_catalog(e.get("binding") or {}) for e in exposes] + if len(resolved) < 2: + return [] + per_table = {"fq_table", "id_columns", "partition_by"} + if all(r.catalog_impl == GLUE_CATALOG_IMPL for r in resolved): + per_table.add("warehouse") + differs: list[str] = [] + for f in dataclass_fields(ResolvedIcebergCatalog): + if f.name in per_table: + continue + if f.name == "extra_catalog_props": + # Named by the binding setting each property comes from. + props = sorted({k for r in resolved for k in r.extra_catalog_props}) + sources = [ + _CATALOG_PROP_SOURCES.get(k, "catalog properties") + for k in props + if any( + r.extra_catalog_props.get(k) != resolved[0].extra_catalog_props.get(k) + for r in resolved[1:] + ) + ] + elif any(getattr(r, f.name) != getattr(resolved[0], f.name) for r in resolved[1:]): + sources = [_CATALOG_FIELD_SOURCES.get(f.name, f.name)] + else: + sources = [] + differs.extend(s for s in dict.fromkeys(sources) if s not in differs) + return differs + + +#: Where a catalog that is given a warehouse creates a table it does not have +#: (apache-iceberg-1.10.0: aws/.../dynamodb/DynamoDbCatalog.java:163-186, for +#: a namespace with no ``location`` property; core/.../jdbc/JdbcCatalog.java: +#: 282-284 and 521-526). Both strip the warehouse's trailing slash. +_BUCKET_WAREHOUSE_LAYOUT: Mapping[str, str] = MappingProxyType( + { + "dynamodb": "/.db/
", + "jdbc": "//
", + } +) + + +def _derived_warehouses(exposes: Tuple[Mapping[str, Any], ...]) -> Dict[str, str]: + """``{exposeId: warehouse}`` when ``exposes`` are one DynamoDB or JDBC kind + whose warehouses :func:`_bucket_warehouse` derives (no + ``location.warehouse``) and differ, else ``{}``.""" + resolved = [resolve_iceberg_catalog(e.get("binding") or {}) for e in exposes] + kinds = {r.kind for r in resolved} + if len(kinds) != 1 or not kinds <= BUCKET_WAREHOUSE_KINDS: + return {} + if any( + str(((e.get("binding") or {}).get("location") or {}).get("warehouse") or "").strip() + for e in exposes + ): + return {} + if len({r.warehouse for r in resolved}) < 2: + return {} + return {str(_expose_id(e)): r.warehouse for e, r in zip(exposes, resolved, strict=True)} + + #: Object-store URI schemes, and the Iceberg ``FileIO`` each one needs. THE one #: scheme list: the resolver picks a FileIO from it and the sink validator #: refuses a name-only warehouse that carries one. @@ -380,10 +631,15 @@ def resolve_iceberg_catalog( ``account_ref`` feeds the warehouse bucket fallback on the Glue path (a concrete account id at connector-config time). REST catalogs take an explicit - ``uri`` + ``warehouse`` (catalog name) and don't use it. + ``uri`` + ``warehouse`` (catalog name) and don't use it. DynamoDB and JDBC + take ``location.warehouse``, else derive one from an explicit + ``location.bucket`` (:data:`BUCKET_WAREHOUSE_KINDS`); BigQuery takes the + ``gs://`` URI :func:`iceberg_storage_uri` gives for the bucket with its + ``{{ env.* }}`` templates rendered, plus ``location.project`` + and ``location.region`` as its own catalog properties. """ loc = binding.get("location") or {} - database = loc.get("database") or binding.get("database") + database = iceberg_namespace(binding) or None table = loc.get("table") or binding.get("table") fq_table = f"{database}.{table}" kind = iceberg_catalog_kind(binding, sink) @@ -410,7 +666,26 @@ def resolve_iceberg_catalog( # unknown kind keeps the historic REST fallback, and ``fluid validate`` # refuses it). The FileIO follows the WAREHOUSE scheme so GCS (gs://) and # ADLS (abfss://) work, not just S3 (RFC §6.3 — PR7's REST + GCP profiles). - warehouse = loc.get("warehouse") or "" + warehouse = str(loc.get("warehouse") or "").strip() + region = loc.get("region") + extra: Dict[str, str] = {} + if kind == "bigquery": + # The gs:// storage dbt-bigquery's ``external_volume`` and the GCP IaC + # use, or none, with the bucket's ``{{ env.* }}`` templates rendered: + # a bucket that does not resolve here derives none. ``client.region`` + # is an AWS client property, so the region goes to the catalog's own + # location key instead. + rendered = {**loc, "bucket": _rendered_bucket(loc)} + warehouse = iceberg_storage_uri({"location": rendered}, scheme="gs") + project = str(loc.get("project") or "").strip() + if project: + extra[BIGQUERY_PROJECT_ID] = project + location = str(region or "").strip() + if location: + extra[BIGQUERY_LOCATION] = location + region = None + elif not warehouse and kind in BUCKET_WAREHOUSE_KINDS: + warehouse = _bucket_warehouse(binding) return ResolvedIcebergCatalog( catalog_type=info.runtime_type or kind, warehouse=warehouse, @@ -418,13 +693,87 @@ def resolve_iceberg_catalog( catalog_impl=info.catalog_impl, uri=loc.get("uri"), io_impl=_io_impl_for_warehouse(warehouse), - region=loc.get("region"), + region=region, id_columns=id_columns, partition_by=partition_by, + extra_catalog_props=extra, kind=kind, ) +def _bucket_warehouse(binding: Mapping[str, Any]) -> str: + """The object-store warehouse an explicit ``location.bucket`` names, or ``""``. + + ``:///``, with the bucket and path normalised the way + the Glue row's :func:`get_iceberg_warehouse` normalises them (``path`` + defaults to ``/
/``). The scheme follows the platform: + ``s3`` on AWS, ``gs`` on GCP; any other platform derives nothing. A + missing bucket, or one :func:`_rendered_bucket` cannot render, derives + nothing either, rather than the Glue row's account-derived fallback bucket. + """ + from ..iac.provider_match import canonical_cloud + + loc = binding.get("location") or {} + scheme = _BUCKET_WAREHOUSE_SCHEMES.get(canonical_cloud(binding.get("platform"))) + bucket = _rendered_bucket(loc) + if not scheme or not bucket: + return "" + _bucket, path = normalize_location(loc, account_ref="") + return f"{scheme}://{bucket}/{path}" + + +def _rendered_bucket(loc: Mapping[str, Any]) -> str: + """``location.bucket`` with its ``{{ env.* }}`` templates rendered, or ``""``. + + ``""`` when the bucket is absent, names a variable that is unset or empty + in this process's environment, or holds a template that is not + ``{{ env.* }}``. An empty variable counts as unset: rendering it would + turn ``acme-{{ env.LAKE_ENV }}-lake`` into ``acme--lake``, a bucket the + contract does not name. + """ + raw = str(loc.get("bucket") or "").strip() + if not raw or _unset_env_vars(raw) or "{{" in _ENV_TEMPLATE_RE.sub("", raw): + return "" + return str(resolve_env_templates(raw)).strip() + + +def _unset_env_vars(raw: str) -> Tuple[str, ...]: + """The ``{{ env.* }}`` variables ``raw`` names that are unset or empty here.""" + found = (name.strip() for name in _ENV_TEMPLATE_RE.findall(raw)) + return tuple(dict.fromkeys(name for name in found if not os.environ.get(name))) + + +def unset_bucket_env_vars(binding: Mapping[str, Any], kind: str) -> Tuple[str, ...]: + """The unset or empty variables the ``kind`` sink's warehouse waits on. + + Non-empty when the sink resolver derives ``kind``'s warehouse from + ``location.bucket`` and that bucket names ``{{ env.* }}`` variables that + are unset or empty in this process's environment: DynamoDB and JDBC on + aws or gcp with no ``location.warehouse``, and BigQuery with no + scheme-qualified ``location.warehouse``. The bucket is explicit, so the + warehouse derives wherever those variables are set, but not here. ``()`` + when every variable the bucket names has a value, for an absent bucket, + another kind or platform, a warehouse the binding sets, or a bucket + holding a template that is not ``{{ env.* }}`` (nothing resolves that one). + """ + from ..iac.provider_match import canonical_cloud + + loc = binding.get("location") or {} + raw = str(loc.get("bucket") or "") + warehouse = str(loc.get("warehouse") or "").strip() + if kind in BUCKET_WAREHOUSE_KINDS: + reads_bucket = not warehouse and ( + canonical_cloud(binding.get("platform")) in _BUCKET_WAREHOUSE_SCHEMES + ) + else: + # :func:`iceberg_storage_uri` reads the bucket unless the warehouse + # carries a scheme. + reads_bucket = kind == "bigquery" and not warehouse.startswith(_WAREHOUSE_SCHEMES) + if not raw or not reads_bucket or "{{" in _ENV_TEMPLATE_RE.sub("", raw): + return () + return _unset_env_vars(raw) + + # --------------------------------------------------------------------------- # Object-store warehouse URI # --------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 3e454f8d..fa6dba50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,11 @@ dependencies = [ # the sampling capability via that kwarg so the floor must include it. # - The gateway depends on the lifespan API + ``mcp.shared.memory`` # test fixtures introduced in 1.20. - # 1.20+ has all of the above. Pinned by the mcp-sdk-drift CI job. + # - 1.20 and 1.21 import ``eval_type_backport`` from + # ``pydantic._internal._typing_extra``, which pydantic 2.14 removed, + # so ``mcp.server.fastmcp`` fails to import with current pydantic. + # 1.22 is the first 1.x that imports with pydantic 2.14. + # 1.22+ has all of the above. Pinned by the mcp-sdk-drift CI job. # Ceiling ``<3.0`` since Phase 2 (2026-09-14). The codebase is # DUAL-SUPPORT (1.x and 2.x) via the fluid_build/_mcp_compat.py seam, and # widening the ceiling is what makes 2.x BLOCKING: the drift job's @@ -142,7 +146,7 @@ dependencies = [ # AGENT_IDE.md "Related work" section + the gateway commit msg): # docs: https://modelcontextprotocol.github.io/python-sdk/ # repo: https://github.com/modelcontextprotocol/python-sdk - "mcp>=1.20,<3.0", + "mcp>=1.22,<3.0", # (The MCP output-port rate limiter is an in-process monotonic-clock # deque sliding window in SessionState.check_rate_limit — no dependency. # PyrateLimiter was dropped: its Limiter spins a background "leaker" diff --git a/tests/build_runners/test_fw2_catalog_table_dispatch.py b/tests/build_runners/test_fw2_catalog_table_dispatch.py new file mode 100644 index 00000000..0910d96e --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_dispatch.py @@ -0,0 +1,395 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An env-templated bucket, through the acquisition dispatcher a run goes through. + +``base._execute_acquisition_build`` resolves every ``{{ env.X }}`` before the +runner sees the contract, and a missing or empty ``X`` becomes ``""``. The +Iceberg sink preflight read only that resolved contract, so +``acme-{{ env.LAKE_ENV }}-lake`` with ``LAKE_ENV`` unset reached it as +``acme--lake``: the preflight passed and the sink was pushed a warehouse in a +bucket the contract does not name. A fully templated bucket reached it as +``""`` and was refused with a message asking for the bucket the contract sets. +The BigQuery deriver did not render the template at all, and pushed a literal +``gs://{{ env.DATA_LAKE_BUCKET }}`` warehouse even with the variable set. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +from fluid_build.build_runners import base +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + +_VAR = "FW2_LAKE_ENV" +_FULL = "{{ env.%s }}" % _VAR +_PARTIAL = "acme-{{ env.%s }}-lake" % _VAR + + +def _binding(catalog: str, bucket: str, **location: Any) -> Dict[str, Any]: + platform = "gcp" if catalog == "bigquery" else "aws" + loc: Dict[str, Any] = {"database": "streaming", "table": "orders", "catalog": catalog} + loc["bucket"] = bucket + if catalog == "bigquery": + loc["project"] = "acme-proj" + else: + loc["region"] = "eu-west-1" + loc.update(location) + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _contract( + binding: Dict[str, Any], + engine: str, + *, + auto_create: bool = False, + overrides: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + if auto_create: + props["kafka-connect"]["streamingSink"] = {"autoCreate": True} + if overrides: + props["kafka-connect"]["iceberg_catalog_overrides"] = dict(overrides) + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": binding, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + ], + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": ["orders"], + "properties": props, + } + ], + } + + +class _FakeConnect: + """Records the connector configs the Kafka Connect runner pushes.""" + + pushed: List[Tuple[str, Dict[str, Any]]] = [] + + def __init__(self, *_a: Any, **_k: Any) -> None: + pass + + def get_connector(self, _name: str) -> None: + return None + + def create_connector(self, name: str, cfg: Dict[str, Any]) -> None: + type(self).pushed.append((name, dict(cfg))) + + update_config = create_connector + + def __getattr__(self, _name: str) -> Any: + return lambda *a, **k: {"connector": {"state": "RUNNING"}, "tasks": [{"state": "RUNNING"}]} + + +@pytest.fixture +def fake_connect(monkeypatch): + from fluid_build.build_runners.kafka_connect import runner as kc_runner + + _FakeConnect.pushed = [] + monkeypatch.setattr(kc_runner, "KafkaConnectRestClient", _FakeConnect) + return _FakeConnect.pushed + + +def _set(monkeypatch, value: Optional[str]) -> None: + if value is None: + monkeypatch.delenv(_VAR, raising=False) + else: + monkeypatch.setenv(_VAR, value) + + +def _dispatch(contract: Dict[str, Any], tmp_path: Path, monkeypatch) -> int: + monkeypatch.chdir(tmp_path) + return base._execute_acquisition_build(contract["builds"][0], contract, tmp_path, dry_run=False) + + +def _sink_warehouse(pushed: List[Tuple[str, Dict[str, Any]]]) -> Optional[str]: + sinks = [cfg for name, cfg in pushed if name.endswith("-sink")] + assert len(sinks) == 1, pushed + return sinks[0].get("iceberg.catalog.warehouse") + + +def _props_path(tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / "bronze.o" / "b1" / "application.properties" + + +def _preflight_failure(caplog) -> str: + # The runner returns 1; the message is in its error log. + text = "\n".join(r.getMessage() for r in caplog.records) + assert "iceberg sink preflight failed" in text, text + return text + + +# --------------------------------------------------------------------------- +# Kafka Connect, through the dispatcher +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc", "bigquery"]) +@pytest.mark.parametrize("bucket", [_FULL, _PARTIAL], ids=["full", "partial"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_dispatch_refuses_a_bucket_that_does_not_resolve( + fake_connect, tmp_path, monkeypatch, caplog, catalog, bucket, value +): + _set(monkeypatch, value) + extra = {"uri": "jdbc:postgresql://pg:5432/iceberg"} if catalog == "jdbc" else {} + # A Kafka Connect BigQuery sink reads the warehouse only to auto-create + # tables, so it is refused only with auto-create on. + binding = _binding(catalog, bucket, **extra) + contract = _contract(binding, "kafka-connect", auto_create=catalog == "bigquery") + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 1 + # Nothing reached the cluster: no sink in another bucket, no source. + assert fake_connect == [] + text = _preflight_failure(caplog) + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert repr(bucket) in text + assert "requires binding.location.warehouse" not in text + # `fluid validate` only warns about this, so the refusal does not send + # the operator there. + assert "see `fluid validate`" not in text + + +@pytest.mark.parametrize( + "catalog, bucket, expected", + [ + ("dynamodb", _PARTIAL, "s3://acme-prod-lake/streaming/orders/"), + ("dynamodb", _FULL, "s3://prod/streaming/orders/"), + ("bigquery", _PARTIAL, "gs://acme-prod-lake"), + ("bigquery", _FULL, "gs://prod"), + ], + ids=["ddb-partial", "ddb-full", "bq-partial", "bq-full"], +) +def test_kc_dispatch_derives_from_the_resolved_bucket( + fake_connect, tmp_path, monkeypatch, catalog, bucket, expected +): + _set(monkeypatch, "prod") + rc = _dispatch(_contract(_binding(catalog, bucket), "kafka-connect"), tmp_path, monkeypatch) + assert rc == 0 + assert _sink_warehouse(fake_connect) == expected + + +def test_kc_dispatch_a_set_warehouse_ignores_the_bucket_variable( + fake_connect, tmp_path, monkeypatch +): + _set(monkeypatch, None) + binding = _binding("dynamodb", _PARTIAL, warehouse="s3://ops/wh/") + assert _dispatch(_contract(binding, "kafka-connect"), tmp_path, monkeypatch) == 0 + assert _sink_warehouse(fake_connect) == "s3://ops/wh/" + + +def test_the_dispatcher_scope_ends_with_the_run(fake_connect, tmp_path, monkeypatch): + # Outside the dispatcher the preflight reads the contract it is handed. + _set(monkeypatch, None) + contract = _contract(_binding("dynamodb", _PARTIAL), "kafka-connect") + assert _dispatch(contract, tmp_path, monkeypatch) == 1 + resolved = base._resolve_env_placeholders(contract) + assert iceberg_sink_preflight(resolved, "b1") is None + + +# --------------------------------------------------------------------------- +# Debezium Server (embedded), through the dispatcher +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bucket", [_FULL, _PARTIAL], ids=["full", "partial"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_debezium_dispatch_refuses_a_bucket_that_does_not_resolve( + tmp_path, monkeypatch, caplog, bucket, value +): + _set(monkeypatch, value) + contract = _contract(_binding("dynamodb", bucket), "debezium") + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 1 + assert not _props_path(tmp_path).exists() + text = _preflight_failure(caplog) + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert "acme--lake" not in text + + +def test_debezium_dispatch_derives_from_the_resolved_bucket(tmp_path, monkeypatch): + _set(monkeypatch, "prod") + # No server binary on PATH: the runner fails after writing the config. + _dispatch(_contract(_binding("dynamodb", _PARTIAL), "debezium"), tmp_path, monkeypatch) + text = _props_path(tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://acme-prod-lake/streaming/orders/" in text + + +# --------------------------------------------------------------------------- +# BigQuery: the deriver renders the bucket, in all three states +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value, warehouse", + [("mylake", "gs://mylake"), ("", ""), (None, "")], + ids=["set", "empty", "unset"], +) +@pytest.mark.parametrize("auto_create", [False, True], ids=["no-autocreate", "autocreate"]) +def test_bigquery_env_bucket_derive_and_validate(monkeypatch, value, warehouse, auto_create): + _set(monkeypatch, value) + binding = _binding("bigquery", _FULL) + resolved = resolve_iceberg_catalog(binding) + assert resolved.warehouse == warehouse + assert "{{" not in resolved.warehouse + assert resolved.extra_catalog_props["gcp.bigquery.project-id"] == "acme-proj" + + contract = _contract(binding, "kafka-connect", auto_create=auto_create) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + named = [w for w in warnings if _VAR in w] + refusal = iceberg_sink_preflight(contract, "b1") + if value: + assert named == [] and refusal is None + assert not [w for w in warnings if "no gs:// warehouse" in w], warnings + else: + # One warning naming the variable, not the generic no-warehouse one. + assert len(named) == 1, warnings + assert f"{_VAR} is unset or empty here" in named[0] + assert not [w for w in warnings if "no gs:// warehouse" in w], warnings + # The run refuses only when the sink would read the warehouse. + assert ("the runner refuses the build" in named[0]) is auto_create + if auto_create: + assert refusal is not None + assert f"{_VAR} is unset or empty in the runner's environment" in refusal + else: + assert refusal is None + + +@pytest.mark.parametrize("bucket", [_FULL, _PARTIAL], ids=["full", "partial"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_bigquery_without_autocreate_runs_and_names_no_other_bucket( + fake_connect, tmp_path, monkeypatch, caplog, bucket, value +): + _set(monkeypatch, value) + contract = _contract(_binding("bigquery", bucket), "kafka-connect") + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 0, caplog.text + sinks = [cfg for _name, cfg in fake_connect if cfg.get("iceberg.catalog.type") == "bigquery"] + assert len(sinks) == 1, fake_connect + # No warehouse in a bucket the contract does not name (acme--lake, gs://). + pushed = sinks[0].get("iceberg.catalog.warehouse", "") + assert pushed in ("",), sinks[0] + + +def test_bigquery_partial_bucket_with_an_empty_variable_derives_none(monkeypatch): + _set(monkeypatch, "") + assert resolve_iceberg_catalog(_binding("bigquery", _PARTIAL)).warehouse == "" + + +def test_bigquery_gs_warehouse_wins_over_an_unresolved_bucket(monkeypatch): + _set(monkeypatch, None) + binding = _binding("bigquery", _FULL, warehouse="gs://wh/x") + assert resolve_iceberg_catalog(binding).warehouse == "gs://wh/x" + errors, warnings = validate_iceberg_sink(_contract(binding, "kafka-connect")) + assert errors == [] and not [w for w in warnings if _VAR in w], warnings + + +def test_dynamodb_partial_bucket_with_an_empty_variable_derives_none(monkeypatch): + # Rendering it would give s3://acme--lake/..., a bucket the contract does + # not name; validate warns and names the variable instead. + _set(monkeypatch, "") + binding = _binding("dynamodb", _PARTIAL) + assert resolve_iceberg_catalog(binding).warehouse == "" + errors, warnings = validate_iceberg_sink(_contract(binding, "kafka-connect")) + assert errors == [] + assert any(f"{_VAR} is unset or empty here" in w for w in warnings), warnings + + +# --------------------------------------------------------------------------- +# Debezium BigQuery with no warehouse: the server does not boot +# --------------------------------------------------------------------------- + + +def test_debezium_bigquery_no_warehouse_warning_says_the_server_refuses_to_boot(): + binding = _binding("bigquery", "") + binding["location"].pop("bucket") + errors, warnings = validate_iceberg_sink(_contract(binding, "debezium")) + assert errors == [] + hit = [w for w in warnings if "no gs:// warehouse" in w] + assert len(hit) == 1, warnings + assert "refuses to boot" in hit[0] + assert "debezium.sink.iceberg.warehouse" in hit[0] + assert "tables that exist need none" not in hit[0] + + +def test_kafka_connect_bigquery_no_warehouse_warning_is_unchanged(): + binding = _binding("bigquery", "") + binding["location"].pop("bucket") + warnings = validate_iceberg_sink(_contract(binding, "kafka-connect"))[1] + hit = [w for w in warnings if "no gs:// warehouse" in w] + assert len(hit) == 1 and "without auto-create, tables that exist need none" in hit[0] + + +@pytest.mark.parametrize( + "catalog, warehouse", + [ + ("dynamodb", "s3://ops-lake/wh"), + ("jdbc", "s3://ops-lake/wh"), + ("bigquery", "gs://ops-lake/wh"), + ], +) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_dispatch_keeps_an_override_warehouse( + fake_connect, tmp_path, monkeypatch, caplog, catalog, warehouse, value +): + _set(monkeypatch, value) + extra = {"uri": "jdbc:postgresql://pg:5432/iceberg"} if catalog == "jdbc" else {} + contract = _contract( + _binding(catalog, _FULL, **extra), + "kafka-connect", + overrides={"iceberg.catalog.warehouse": warehouse}, + ) + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 0, caplog.text + sinks = [cfg for _name, cfg in fake_connect if "iceberg.tables" in cfg] + assert len(sinks) == 1, fake_connect + assert sinks[0].get("iceberg.catalog.warehouse") == warehouse, sinks[0] diff --git a/tests/build_runners/test_fw2_catalog_table_envbucket.py b/tests/build_runners/test_fw2_catalog_table_envbucket.py new file mode 100644 index 00000000..ef883761 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_envbucket.py @@ -0,0 +1,285 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An env-templated bucket is an explicit bucket, set or not where validate runs. + +DynamoDB and JDBC derive their warehouse from ``location.bucket``. A bucket +written as ``{{ env.DATA_LAKE_BUCKET }}`` (the form +``examples/aws-glue-data-lake/contract-iceberg.fluid.yaml`` uses) derives +nothing in a shell without that variable, and check 4 used to treat it as +absent: ``fluid validate`` in such a shell (a CI job, say) refused the +contract and asked for the bucket it already sets, while the same command with +the variable set accepted it. Validate now warns and names the variable; the +runner's preflight, in the environment the sink config is derived in, still +refuses when it does not resolve there. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog, unset_bucket_env_vars + +pytestmark = [pytest.mark.unit] + +_VAR = "FW2_DATA_LAKE_BUCKET" +_TEMPLATE = "{{ env.%s }}" % _VAR +_JDBC_URI = "jdbc:postgresql://pg:5432/iceberg" + + +def _binding(catalog: str, platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc = {"catalog": catalog, "database": "streaming", "table": "orders", **location} + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _contract( + binding: Dict[str, Any], *, engine: str = "kafka-connect", props: Optional[Dict] = None +) -> Dict[str, Any]: + build_props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append"}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + build_props["debezium"] = props or { + "deployment": {"mode": "embedded"}, + "server": {"sink": {}}, + } + else: + build_props["kafka-connect"] = props or {} + return { + "id": "bronze.orders_stream", + "builds": [{"id": "ingest", "engine": engine, "properties": build_props}], + "exposes": [{"exposeId": "orders", "kind": "table", "binding": binding}], + } + + +_TEMPLATED = [ + _binding("dynamodb", bucket=_TEMPLATE, region="eu-west-1"), + _binding("jdbc", bucket=_TEMPLATE, region="eu-west-1", uri=_JDBC_URI), +] +_IDS = ["dynamodb", "jdbc"] + + +@pytest.fixture +def unset(monkeypatch): + monkeypatch.delenv(_VAR, raising=False) + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_unresolved_bucket_template_warns_at_validate(unset, binding, engine): + kind = binding["location"]["catalog"] + errors, warnings = validate_iceberg_sink(_contract(binding, engine=engine)) + assert errors == [] + hit = [w for w in warnings if _VAR in w] + assert len(hit) == 1, warnings + assert f"the {kind} catalog's warehouse derives from binding.location.bucket" in hit[0] + assert repr(_TEMPLATE) in hit[0] + assert f"{_VAR} is unset or empty here" in hit[0] + # It does not ask for the bucket the contract already sets. + assert "requires binding.location.warehouse" not in hit[0] + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_unresolved_bucket_template_is_refused_by_the_preflight(unset, binding, engine): + refusal = iceberg_sink_preflight(_contract(binding, engine=engine), "ingest") + assert refusal is not None + assert f"{_VAR} is unset or empty in the runner's environment" in refusal + assert "binding.location.warehouse" in refusal + + +def test_the_preflight_leaves_validate_in_validate_mode(unset): + contract = _contract(_TEMPLATED[0]) + assert iceberg_sink_preflight(contract, "ingest") is not None + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] and any(_VAR in w for w in warnings) + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_resolved_bucket_template_validates_clean_and_derives(monkeypatch, binding, engine): + monkeypatch.setenv(_VAR, "acme-lake") + contract = _contract(binding, engine=engine) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert not [w for w in warnings if _VAR in w or "warehouse" in w], warnings + assert iceberg_sink_preflight(contract, "ingest") is None + resolved = resolve_iceberg_catalog(binding, account_ref="123456789012") + if engine == "debezium": + warehouse = emit_debezium_iceberg_sink_config(resolved)["warehouse"] + else: + cfg = emit_iceberg_sink_config(resolved, product_id="bronze.orders_stream", topics=["t"]) + warehouse = cfg["iceberg.catalog.warehouse"] + assert warehouse == "s3://acme-lake/streaming/orders/" + + +def test_resolved_bucket_template_on_gcp_derives_gs(monkeypatch): + monkeypatch.setenv(_VAR, "acme-lake") + binding = _binding("jdbc", "gcp", bucket=_TEMPLATE, uri=_JDBC_URI) + assert resolve_iceberg_catalog(binding).warehouse == "gs://acme-lake/streaming/orders/" + + +def test_several_unset_variables_are_all_named(monkeypatch): + monkeypatch.delenv("FW2_A", raising=False) + monkeypatch.delenv("FW2_B", raising=False) + binding = _binding("dynamodb", bucket="{{ env.FW2_A }}-{{ env.FW2_B }}", region="eu-west-1") + assert unset_bucket_env_vars(binding, "dynamodb") == ("FW2_A", "FW2_B") + warnings = validate_iceberg_sink(_contract(binding))[1] + assert any("FW2_A, FW2_B are unset or empty here" in w and "Set them" in w for w in warnings) + + +def test_only_the_variables_without_a_value_are_named(monkeypatch): + monkeypatch.setenv("FW2_A", "acme") + monkeypatch.delenv("FW2_B", raising=False) + binding = _binding("dynamodb", bucket="{{ env.FW2_B }}", region="eu-west-1") + assert unset_bucket_env_vars(binding, "dynamodb") == ("FW2_B",) + binding = _binding("dynamodb", bucket="{{ env.FW2_A }}", region="eu-west-1") + assert unset_bucket_env_vars(binding, "dynamodb") == () + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +def test_an_empty_variable_is_treated_like_an_unset_one(monkeypatch, binding): + # An empty value resolves the bucket to "", which derives no warehouse; the + # message still names the variable instead of asking for a bucket. + monkeypatch.setenv(_VAR, "") + contract = _contract(binding) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert any(f"{_VAR} is unset or empty here" in w for w in warnings), warnings + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None + assert f"{_VAR} is unset or empty in the runner's environment" in refusal + assert "requires binding.location.warehouse" not in refusal + + +@pytest.mark.parametrize( + "binding", + [ + # nothing resolves a template that is not {{ env.* }} + _binding("dynamodb", bucket="{{ var.lake }}", region="eu-west-1"), + _binding("dynamodb", bucket="{{ env.%s }}-{{ var.x }}" % _VAR, region="eu-west-1"), + # no scheme to derive with off aws / gcp, template or not + _binding("dynamodb", "local", bucket=_TEMPLATE), + ], + ids=["non-env-template", "mixed-template", "local-platform"], +) +def test_a_bucket_that_cannot_derive_anywhere_stays_a_hard_error(unset, binding): + assert unset_bucket_env_vars(binding, "dynamodb") == () + errors = validate_iceberg_sink(_contract(binding))[0] + assert any("dynamodb catalog requires binding.location.warehouse" in e for e in errors) + + +def test_an_override_warehouse_needs_no_bucket_warning(unset): + contract = _contract( + _TEMPLATED[0], + props={"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "s3://ops/wh/"}}, + ) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] and not [w for w in warnings if _VAR in w], warnings + assert iceberg_sink_preflight(contract, "ingest") is None + + +def test_glue_with_an_unresolved_template_is_unchanged(unset): + binding = _binding("glue", bucket=_TEMPLATE, region="eu-west-1") + contract = _contract(binding) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None + + +_CONTRACT_YAML = ( + """\ +fluidVersion: 0.7.6 +kind: DataProduct +domain: sales +metadata: + layer: Bronze + owner: + team: dp +id: bronze.o +name: o +exposes: +- exposeId: orders + kind: table + binding: + platform: aws + format: iceberg + location: + database: streaming + table: orders + catalog: dynamodb + bucket: '{{ env.%s }}' + region: eu-west-1 + contract: + schema: + - name: id + type: integer +builds: +- id: b1 + pattern: acquisition + engine: kafka-connect + outputs: + - orders + properties: + source: + kind: postgres + mode: incremental_append + streams: + - public.o + sink: + format: iceberg + kafka-connect: + deployment: + mode: bring-your-own + server_url: http://c:8083 +""" + % _VAR +) + + +@pytest.mark.parametrize("value", [None, "", "acme-lake"], ids=["unset", "empty", "set"]) +def test_cli_validate_accepts_the_contract_with_or_without_the_variable(tmp_path, value): + path = tmp_path / "kc_ddb_envbucket.fluid.yaml" + path.write_text(textwrap.dedent(_CONTRACT_YAML), encoding="utf-8") + env = os.environ.copy() + env.pop(_VAR, None) + if value is not None: + env[_VAR] = value + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + env["FLUID_NONINTERACTIVE"] = "1" + proc = subprocess.run( + [sys.executable, "-m", "fluid_build", "validate", str(path)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + timeout=120, + ) + out = proc.stdout + proc.stderr + assert proc.returncode == 0, out + assert "requires binding.location.warehouse" not in out + assert (_VAR in out) is (not value), out diff --git a/tests/build_runners/test_fw2_catalog_table_overrides.py b/tests/build_runners/test_fw2_catalog_table_overrides.py new file mode 100644 index 00000000..a43b3a57 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_overrides.py @@ -0,0 +1,235 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An override or a hand-written config can supply what a catalog needs to start. + +The DynamoDB / JDBC warehouse and the BigQuery ``gcp.bigquery.project-id`` are +hard requirements of the sink, but the runner merges operator maps into the +config it pushes, so a key set there reaches the worker. Validate and the +preflight used to read only ``binding.location`` for them and refused configs +whose pushed form carried the key. The prefix is the runtime's: Kafka Connect +reads ``iceberg.catalog.``, Debezium Server takes the bare ````. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import pytest + +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import DYNAMODB_CATALOG_IMPL, resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + +_JDBC_URI = "jdbc:postgresql://pg:5432/iceberg" + + +def _binding(catalog: str, platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc = {"catalog": catalog, "database": "streaming", "table": "orders", **location} + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _kc_contract(binding: Dict[str, Any], kc: Dict[str, Any]) -> Dict[str, Any]: + return _contract(binding, "kafka-connect", kc) + + +def _dbz_contract(binding: Dict[str, Any], server_sink: Dict[str, Any]) -> Dict[str, Any]: + return _contract( + binding, "debezium", {"deployment": {"mode": "embedded"}, "server": {"sink": server_sink}} + ) + + +def _contract(binding: Dict[str, Any], engine: str, engine_props: Dict[str, Any]): + return { + "id": "bronze.orders_stream", + "builds": [ + { + "id": "ingest", + "engine": engine, + "properties": { + "source": {"kind": "postgres", "mode": "incremental_append"}, + "sink": {"format": "iceberg"}, + engine: engine_props, + }, + } + ], + "exposes": [{"exposeId": "orders", "kind": "table", "binding": binding}], + } + + +def _accepted(contract: Dict[str, Any]) -> None: + errors = validate_iceberg_sink(contract)[0] + assert errors == [], errors + assert iceberg_sink_preflight(contract, "ingest") is None + + +def _refused(contract: Dict[str, Any], needle: str) -> None: + errors = validate_iceberg_sink(contract)[0] + assert any(needle in e for e in errors), errors + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None and needle in refusal + + +_NO_WH = { + "dynamodb": _binding("dynamodb", region="eu-west-1"), + "jdbc": _binding("jdbc", region="eu-west-1", uri=_JDBC_URI), +} + + +# ── warehouse: DynamoDB / JDBC ────────────────────────────────────────────── + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_kafka_connect_catalog_override_warehouse_is_accepted(kind): + kc = {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "s3://ops/wh"}} + _accepted(_kc_contract(_NO_WH[kind], kc)) + # ...and it is what the runner pushes. + resolved = resolve_iceberg_catalog(_NO_WH[kind], account_ref="") + cfg = emit_iceberg_sink_config(resolved, product_id="p", topics=["t"], kc_props=kc) + assert cfg["iceberg.catalog.warehouse"] == "s3://ops/wh" + + +def test_kafka_connect_hand_written_config_with_a_warehouse_is_accepted(): + kc = { + "sink_connector_config": { + "connector.class": "x", + "iceberg.catalog.catalog-impl": DYNAMODB_CATALOG_IMPL, + "iceberg.catalog.warehouse": "s3://ops/wh", + } + } + _accepted(_kc_contract(_NO_WH["dynamodb"], kc)) + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("enabled", [True, False], ids=["derived", "hand-written"]) +def test_debezium_server_sink_config_warehouse_is_accepted(kind, enabled): + server_sink = {"iceberg_sink_enabled": enabled, "config": {"warehouse": "s3://ops/wh"}} + if not enabled and kind == "dynamodb": + server_sink["config"]["catalog-impl"] = DYNAMODB_CATALOG_IMPL + if not enabled and kind == "jdbc": + server_sink["config"]["type"] = "jdbc" + _accepted(_dbz_contract(_NO_WH[kind], server_sink)) + + +@pytest.mark.parametrize( + "kc", + [ + {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": ""}}, + # Kafka Connect does not read a bare ``warehouse`` as a catalog property + {"iceberg_catalog_overrides": {"warehouse": "s3://ops/wh"}}, + {"iceberg_catalog_overrides": {"iceberg.catalog.client.region": "eu-west-1"}}, + ], + ids=["empty", "unprefixed", "other-key"], +) +def test_an_override_that_sets_no_catalog_warehouse_is_still_refused(kc): + _refused( + _kc_contract(_NO_WH["dynamodb"], kc), "dynamodb catalog requires binding.location.warehouse" + ) + + +def test_debezium_prefixed_warehouse_does_not_count(): + server_sink = {"iceberg_sink_enabled": True, "config": {"iceberg.catalog.warehouse": "s3://o"}} + _refused( + _dbz_contract(_NO_WH["dynamodb"], server_sink), + "dynamodb catalog requires binding.location.warehouse", + ) + + +def test_an_override_warehouse_does_not_satisfy_the_jdbc_uri(): + binding = _binding("jdbc", region="eu-west-1") + kc = {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "s3://ops/wh"}} + errors = validate_iceberg_sink(_kc_contract(binding, kc))[0] + assert [e for e in errors if "requires binding.location" in e] == [ + "iceberg sink (build 'ingest'): jdbc catalog requires binding.location.uri" + ] + + +def test_the_refusal_names_the_override_remedy(): + errors = validate_iceberg_sink(_kc_contract(_NO_WH["dynamodb"], {}))[0] + hit = [e for e in errors if "requires binding.location.warehouse" in e] + assert len(hit) == 1 and "in an override" in hit[0] + + +# ── project: BigQuery ─────────────────────────────────────────────────────── + +_BQ_NO_PROJECT = _binding("bigquery", "gcp", bucket="lake") + + +def test_kafka_connect_catalog_override_project_is_accepted(): + kc = {"iceberg_catalog_overrides": {"iceberg.catalog.gcp.bigquery.project-id": "p"}} + _accepted(_kc_contract(_BQ_NO_PROJECT, kc)) + + +def test_kafka_connect_hand_written_project_is_accepted(): + kc = { + "sink_connector_config": { + "iceberg.catalog.type": "bigquery", + "iceberg.catalog.gcp.bigquery.project-id": "p", + } + } + _accepted(_kc_contract(_BQ_NO_PROJECT, kc)) + + +def test_debezium_server_sink_config_project_is_accepted(): + server_sink = {"iceberg_sink_enabled": True, "config": {"gcp.bigquery.project-id": "p"}} + _accepted(_dbz_contract(_BQ_NO_PROJECT, server_sink)) + + +@pytest.mark.parametrize( + "overrides", + [{"iceberg.catalog.gcp.bigquery.project-id": ""}, {"gcp.bigquery.project-id": "p"}], + ids=["empty", "unprefixed"], +) +def test_an_override_that_sets_no_project_is_still_refused(overrides): + _refused( + _kc_contract(_BQ_NO_PROJECT, {"iceberg_catalog_overrides": overrides}), + "bigquery catalog requires binding.location.project", + ) + + +# ── the BigQuery no-warehouse warning ─────────────────────────────────────── + + +def _warehouse_warning(contract: Dict[str, Any]) -> str: + hit = [w for w in validate_iceberg_sink(contract)[1] if "no gs:// warehouse" in w] + assert len(hit) == 1, hit + return hit[0] + + +def test_kafka_connect_warning_says_auto_create_fails_even_in_an_existing_dataset(): + # apache-iceberg-1.10.0: IcebergWriterFactory.autoCreateTable calls + # createNamespace before createTable, and BigQueryMetastoreCatalog's + # createNamespace refuses without a warehouse. + binding = _binding("bigquery", "gcp", project="acme-proj", region="EU") + warning = _warehouse_warning(_kc_contract(binding, {"streamingSink": {"autoCreate": True}})) + assert "createNamespace" in warning + assert "table auto-creation fails even in a dataset that exists" in warning + assert "default storage location URI" not in warning + + +def test_debezium_warning_says_the_server_refuses_to_boot(): + # memiiso/debezium-server-iceberg 1.2.0.Final IcebergConfig.java:44-45 + # declares debezium.sink.iceberg.warehouse as a String with no default, so + # the server does not start without one, whether or not the tables exist. + binding = _binding("bigquery", "gcp", project="acme-proj") + warning = _warehouse_warning(_dbz_contract(binding, {})) + assert "refuses to boot" in warning + assert "debezium.sink.iceberg.warehouse" in warning + assert "default storage location URI" not in warning + assert "createNamespace" not in warning diff --git a/tests/build_runners/test_fw2_catalog_table_plan_path.py b/tests/build_runners/test_fw2_catalog_table_plan_path.py new file mode 100644 index 00000000..f371ebe6 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_plan_path.py @@ -0,0 +1,305 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An env-templated bucket on the plan path, and a sink that never reads it. + +``fluid apply .json --mode amend-and-build`` runs +``run_builds_from_args(args, plan_data=...)``, which resolves the plan's +embedded contract with ``resolve_contract_env_templates`` before dispatching. +That resolver renders an empty variable to ``""``, so +``acme-{{ env.LAKE_ENV }}-lake`` reached the dispatcher as ``acme--lake`` and +the sink was pushed a warehouse in a bucket the contract does not name. + +A hand-written (non-deriving) sink config never reads the bucket, but the +preflight refused it when a bucket variable was unset. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +from fluid_build.build_runners import base +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) + +pytestmark = [pytest.mark.unit] + +_VAR = "FW2_PLAN_LAKE_ENV" +_PARTIAL = "acme-{{ env.%s }}-lake" % _VAR +_FULL = "{{ env.%s }}" % _VAR + + +def _contract( + engine: str, catalog: str, bucket: str, *, auto_create: bool = False +) -> Dict[str, Any]: + platform = "gcp" if catalog == "bigquery" else "aws" + loc: Dict[str, Any] = { + "database": "streaming", + "table": "orders", + "catalog": catalog, + "bucket": bucket, + } + if catalog == "bigquery": + loc["project"] = "acme-proj" + else: + loc["region"] = "eu-west-1" + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + if auto_create: + props["kafka-connect"]["streamingSink"] = {"autoCreate": True} + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": {"platform": platform, "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + ], + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": ["orders"], + "properties": props, + } + ], + } + + +class _FakeConnect: + """Records the connector configs the Kafka Connect runner pushes.""" + + pushed: List[Tuple[str, Dict[str, Any]]] = [] + + def __init__(self, *_a: Any, **_k: Any) -> None: + pass + + def get_connector(self, _name: str) -> None: + return None + + def create_connector(self, name: str, cfg: Dict[str, Any]) -> None: + type(self).pushed.append((name, dict(cfg))) + + update_config = create_connector + + def __getattr__(self, _name: str) -> Any: + return lambda *a, **k: {"connector": {"state": "RUNNING"}, "tasks": [{"state": "RUNNING"}]} + + +@pytest.fixture +def fake_connect(monkeypatch): + from fluid_build.build_runners.kafka_connect import runner as kc_runner + + _FakeConnect.pushed = [] + monkeypatch.setattr(kc_runner, "KafkaConnectRestClient", _FakeConnect) + return _FakeConnect.pushed + + +def _set(monkeypatch, value: Optional[str]) -> None: + if value is None: + monkeypatch.delenv(_VAR, raising=False) + else: + monkeypatch.setenv(_VAR, value) + + +def _run_plan(contract: Dict[str, Any], tmp_path: Path, monkeypatch) -> int: + """``run_builds_from_args`` as ``fluid apply .json`` calls it.""" + monkeypatch.chdir(tmp_path) + plan_data = {"contract": contract} + plan = tmp_path / "plan.json" + plan.write_text(json.dumps(plan_data), encoding="utf-8") + args = argparse.Namespace( + contract=str(plan), + build_id=None, + dry_run=False, + env=None, + fail_fast=False, + sample_rows=None, + ) + return base.run_builds_from_args( + args, logging.getLogger("fw2.plan"), force_run=True, plan_data=plan_data + ) + + +def _log(caplog) -> str: + return "\n".join(r.getMessage() for r in caplog.records) + + +def _props_path(tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / "bronze.o" / "b1" / "application.properties" + + +# --------------------------------------------------------------------------- +# The plan path reads the bucket template as the plan wrote it +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc", "bigquery"]) +@pytest.mark.parametrize("bucket", [_PARTIAL, _FULL], ids=["partial", "full"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_plan_path_refuses_a_bucket_that_does_not_resolve( + fake_connect, tmp_path, monkeypatch, caplog, catalog, bucket, value +): + _set(monkeypatch, value) + # A Kafka Connect BigQuery sink is refused only with auto-create on. + contract = _contract("kafka-connect", catalog, bucket, auto_create=catalog == "bigquery") + if catalog == "jdbc": + contract["exposes"][0]["binding"]["location"]["uri"] = "jdbc:postgresql://pg/ice" + with caplog.at_level(logging.INFO): + rc = _run_plan(contract, tmp_path, monkeypatch) + assert rc == 1 + # Nothing reached the cluster: no sink in a bucket the contract does not name. + assert fake_connect == [] + text = _log(caplog) + assert "iceberg sink preflight failed" in text + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert "acme--lake" not in text + + +def test_kc_plan_path_derives_from_the_resolved_bucket(fake_connect, tmp_path, monkeypatch): + _set(monkeypatch, "prod") + rc = _run_plan(_contract("kafka-connect", "dynamodb", _PARTIAL), tmp_path, monkeypatch) + assert rc == 0 + sinks = [cfg for name, cfg in fake_connect if name.endswith("-sink")] + assert len(sinks) == 1 + assert sinks[0]["iceberg.catalog.warehouse"] == "s3://acme-prod-lake/streaming/orders/" + + +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_debezium_plan_path_refuses_a_bucket_that_does_not_resolve( + tmp_path, monkeypatch, caplog, value +): + _set(monkeypatch, value) + with caplog.at_level(logging.INFO): + rc = _run_plan(_contract("debezium", "dynamodb", _PARTIAL), tmp_path, monkeypatch) + assert rc == 1 + assert not _props_path(tmp_path).exists() + text = _log(caplog) + assert "iceberg sink preflight failed" in text + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert "acme--lake" not in text + + +def test_debezium_plan_path_derives_from_the_resolved_bucket(tmp_path, monkeypatch): + _set(monkeypatch, "prod") + # No server binary on PATH: the runner fails after writing the config. + _run_plan(_contract("debezium", "dynamodb", _PARTIAL), tmp_path, monkeypatch) + text = _props_path(tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://acme-prod-lake/streaming/orders/" in text + + +def test_plan_path_matches_a_binding_the_plan_resolver_stripped( + fake_connect, tmp_path, monkeypatch, caplog +): + # ``resolve_contract_env_templates`` strips every string it renders; the + # binding is still found in the plan as written. + _set(monkeypatch, "") + contract = _contract("kafka-connect", "dynamodb", " %s " % _PARTIAL) + with caplog.at_level(logging.INFO): + assert _run_plan(contract, tmp_path, monkeypatch) == 1 + assert fake_connect == [] + assert f"{_VAR} is unset or empty in the runner's environment" in _log(caplog) + + +# --------------------------------------------------------------------------- +# A hand-written sink config never reads the bucket +# --------------------------------------------------------------------------- + + +_HANDWRITTEN_BQ = { + "connector.class": "org.apache.iceberg.connect.IcebergSinkConnector", + "topics": "orders", + "iceberg.tables": "streaming.orders", + "iceberg.catalog.type": "bigquery", + "iceberg.catalog.gcp.bigquery.project-id": "acme-proj", +} + + +def _handwritten_kc(catalog: str = "bigquery") -> Dict[str, Any]: + contract = _contract("kafka-connect", catalog, _FULL) + cfg = dict(_HANDWRITTEN_BQ) + if catalog == "dynamodb": + cfg.pop("iceberg.catalog.type") + cfg.pop("iceberg.catalog.gcp.bigquery.project-id") + cfg["iceberg.catalog.catalog-impl"] = "org.apache.iceberg.aws.dynamodb.DynamoDbCatalog" + cfg["iceberg.catalog.warehouse"] = "s3://ops/wh/" + contract["builds"][0]["properties"]["kafka-connect"]["sink_connector_config"] = cfg + return contract + + +@pytest.mark.parametrize("catalog", ["bigquery", "dynamodb"]) +def test_handwritten_kc_config_ignores_an_unset_bucket_variable( + fake_connect, tmp_path, monkeypatch, catalog +): + _set(monkeypatch, None) + contract = _handwritten_kc(catalog) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert not [w for w in warnings if _VAR in w], warnings + assert iceberg_sink_preflight(contract, "b1") is None + + monkeypatch.chdir(tmp_path) + rc = base._execute_acquisition_build(contract["builds"][0], contract, tmp_path, dry_run=False) + assert rc == 0 + sinks = [cfg for name, cfg in fake_connect if name.endswith("-sink")] + assert len(sinks) == 1 + pushed = {k: v for k, v in sinks[0].items() if k.startswith("iceberg.catalog.")} + written = contract["builds"][0]["properties"]["kafka-connect"]["sink_connector_config"] + assert pushed == {k: v for k, v in written.items() if k.startswith("iceberg.catalog.")} + + +def test_handwritten_debezium_config_ignores_an_unset_bucket_variable(monkeypatch): + _set(monkeypatch, None) + contract = _contract("debezium", "dynamodb", _FULL) + contract["builds"][0]["properties"]["debezium"]["server"]["sink"] = { + "config": { + "catalog-impl": "org.apache.iceberg.aws.dynamodb.DynamoDbCatalog", + "warehouse": "s3://ops/wh/", + } + } + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert not [w for w in warnings if _VAR in w], warnings + assert iceberg_sink_preflight(contract, "b1") is None + + +def test_a_deriving_build_still_waits_on_the_bucket_variable(monkeypatch): + # Control for the tests above: the same contract, derived, is named. + _set(monkeypatch, None) + contract = _contract("kafka-connect", "bigquery", _FULL) + warnings = validate_iceberg_sink(contract)[1] + assert [w for w in warnings if _VAR in w], warnings diff --git a/tests/build_runners/test_fw2_catalog_table_sink.py b/tests/build_runners/test_fw2_catalog_table_sink.py new file mode 100644 index 00000000..2d35f9cf --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_sink.py @@ -0,0 +1,231 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The derived sink configs carry what DynamoDB, JDBC and BigQuery need to start. + +CON-3: the derived dynamodb and jdbc configs had no warehouse, which both +catalogs refuse at initialise, yet validate and the preflight passed. CON-4: +the derived bigquery config had no ``gcp.bigquery.project-id`` (required at +initialise), no warehouse, and an AWS-only ``client.region``. Property names +are from apache/iceberg at apache-iceberg-1.10.0. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + +_JDBC_URI = "jdbc:postgresql://pg:5432/iceberg" + + +def _binding(catalog: str, platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc = {"catalog": catalog, "database": "streaming", "table": "orders", **location} + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _contract( + binding: Dict[str, Any], *, engine: str = "kafka-connect", props: Optional[Dict] = None +) -> Dict[str, Any]: + build_props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append"}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + build_props["debezium"] = props or { + "deployment": {"mode": "embedded"}, + "server": {"sink": {}}, + } + else: + build_props["kafka-connect"] = props or {} + return { + "id": "bronze.orders_stream", + "builds": [{"id": "ingest", "engine": engine, "properties": build_props}], + "exposes": [{"exposeId": "orders", "kind": "table", "binding": binding}], + } + + +def _kc(binding: Dict[str, Any]) -> Dict[str, str]: + resolved = resolve_iceberg_catalog(binding, account_ref="123456789012") + return emit_iceberg_sink_config(resolved, product_id="bronze.orders_stream", topics=["t"]) + + +def _dbz(binding: Dict[str, Any]) -> Dict[str, str]: + return emit_debezium_iceberg_sink_config( + resolve_iceberg_catalog(binding, account_ref="123456789012") + ) + + +# ── CON-3: dynamodb / jdbc ────────────────────────────────────────────────── + +_DERIVABLE = [ + _binding("dynamodb", bucket="acme-lake", region="eu-west-1"), + _binding("jdbc", bucket="acme-lake", region="eu-west-1", uri=_JDBC_URI), +] + + +@pytest.mark.parametrize("binding", _DERIVABLE, ids=["dynamodb", "jdbc"]) +def test_both_runtimes_emit_the_bucket_derived_warehouse(binding): + assert _kc(binding)["iceberg.catalog.warehouse"] == "s3://acme-lake/streaming/orders/" + assert _dbz(binding)["warehouse"] == "s3://acme-lake/streaming/orders/" + + +@pytest.mark.parametrize("binding", _DERIVABLE, ids=["dynamodb", "jdbc"]) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_derivable_warehouse_validates_clean(binding, engine): + contract = _contract(binding, engine=engine) + assert validate_iceberg_sink(contract)[0] == [] + assert iceberg_sink_preflight(contract, "ingest") is None + + +@pytest.mark.parametrize( + "binding", + [ + _binding("dynamodb", region="eu-west-1"), + _binding("jdbc", region="eu-west-1", uri=_JDBC_URI), + # no scheme to derive with off aws / gcp + _binding("dynamodb", "local", bucket="acme-lake"), + ], + ids=["dynamodb", "jdbc", "dynamodb-local"], +) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_no_derivable_warehouse_is_a_hard_error_and_the_preflight_refuses(binding, engine): + kind = binding["location"]["catalog"] + contract = _contract(binding, engine=engine) + errors = validate_iceberg_sink(contract)[0] + hit = [e for e in errors if f"{kind} catalog requires binding.location.warehouse" in e] + assert len(hit) == 1, errors + assert "binding.location.bucket" in hit[0] + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None and "binding.location.warehouse" in refusal + + +def test_jdbc_still_requires_its_uri(): + errors = validate_iceberg_sink(_contract(_binding("jdbc", bucket="acme-lake")))[0] + assert any("jdbc catalog requires binding.location.uri" in e for e in errors), errors + + +def test_an_explicit_warehouse_satisfies_dynamodb_off_aws(): + binding = _binding("dynamodb", "local", warehouse="s3://wh/root") + assert validate_iceberg_sink(_contract(binding))[0] == [] + assert _kc(binding)["iceberg.catalog.warehouse"] == "s3://wh/root" + + +# ── CON-4: bigquery ───────────────────────────────────────────────────────── + + +def _bq(**location: Any) -> Dict[str, Any]: + return _binding("bigquery", "gcp", **location) + + +_FULL_BQ = _bq(project="acme-proj", region="eu", bucket="acme-lake") + + +def test_kafka_connect_bigquery_config_carries_project_location_and_warehouse(): + cfg = _kc(_FULL_BQ) + assert cfg["iceberg.catalog.type"] == "bigquery" + assert cfg["iceberg.catalog.gcp.bigquery.project-id"] == "acme-proj" + assert cfg["iceberg.catalog.gcp.bigquery.location"] == "eu" + assert cfg["iceberg.catalog.warehouse"] == "gs://acme-lake" + assert "iceberg.catalog.client.region" not in cfg + + +def test_debezium_bigquery_config_carries_project_location_and_warehouse(): + cfg = _dbz(_FULL_BQ) + assert cfg["type"] == "bigquery" + assert cfg["gcp.bigquery.project-id"] == "acme-proj" + assert cfg["gcp.bigquery.location"] == "eu" + assert cfg["warehouse"] == "gs://acme-lake" + assert "client.region" not in cfg + + +def test_an_operator_override_still_wins_over_a_derived_bigquery_key(): + resolved = resolve_iceberg_catalog(_FULL_BQ) + cfg = emit_iceberg_sink_config( + resolved, + product_id="p", + topics=["t"], + kc_props={"iceberg_catalog_overrides": {"iceberg.catalog.gcp.bigquery.location": "us"}}, + ) + assert cfg["iceberg.catalog.gcp.bigquery.location"] == "us" + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_bigquery_without_project_is_a_hard_error_and_the_preflight_refuses(engine): + contract = _contract(_bq(bucket="acme-lake"), engine=engine) + errors = validate_iceberg_sink(contract)[0] + hit = [e for e in errors if "bigquery catalog requires binding.location.project" in e] + assert len(hit) == 1, errors + assert "gcp.bigquery.project-id" in hit[0] + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None and "binding.location.project" in refusal + + +def _warehouse_warnings(contract: Dict[str, Any]): + return [w for w in validate_iceberg_sink(contract)[1] if "no gs:// warehouse" in w] + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_bigquery_with_no_derivable_warehouse_warns(engine): + contract = _contract(_bq(project="acme-proj"), engine=engine) + errors = validate_iceberg_sink(contract)[0] + assert errors == [] + hit = _warehouse_warnings(contract) + assert len(hit) == 1 + assert ("debezium.sink.iceberg.warehouse" in hit[0]) == (engine == "debezium") + + +def test_bigquery_with_a_bucket_does_not_warn_about_the_warehouse(): + assert _warehouse_warnings(_contract(_FULL_BQ)) == [] + + +@pytest.mark.parametrize( + "engine, props", + [ + ( + "kafka-connect", + {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "gs://ops/wh"}}, + ), + ( + "debezium", + { + "deployment": {"mode": "embedded"}, + "server": { + "sink": {"iceberg_sink_enabled": True, "config": {"warehouse": "gs://o"}} + }, + }, + ), + ], +) +def test_an_override_warehouse_silences_the_bigquery_warning(engine, props): + contract = _contract(_bq(project="acme-proj"), engine=engine, props=props) + assert _warehouse_warnings(contract) == [] + + +def test_runtime_version_warning_names_the_failure_and_the_remedy(): + warnings = validate_iceberg_sink(_contract(_FULL_BQ))[1] + hit = [w for w in warnings if "iceberg.catalog.type=bigquery" in w] + assert len(hit) == 1 + assert "on that sink the connector fails at start" in hit[0] + assert "Run a sink built from Iceberg >= 1.10" in hit[0] diff --git a/tests/build_runners/test_fw2_catalog_table_whitespace.py b/tests/build_runners/test_fw2_catalog_table_whitespace.py new file mode 100644 index 00000000..8880b1a4 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_whitespace.py @@ -0,0 +1,136 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A whitespace-only ``location.warehouse`` counts as unset. + +``' '`` used to reach the sink as its warehouse. Now a DynamoDB or JDBC sink +derives the warehouse from the bucket, or validate and the preflight refuse. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + + +def _binding(catalog: str, **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = {"database": "streaming", "table": "orders", "catalog": catalog} + loc["region"] = "eu-west-1" + if catalog == "jdbc": + loc["uri"] = "jdbc:postgresql://pg:5432/iceberg" + loc.update(location) + return {"platform": "aws", "format": "iceberg", "location": loc} + + +def _contract(binding: Dict[str, Any], engine: str) -> Dict[str, Any]: + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": binding, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + ], + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": ["orders"], + "properties": props, + } + ], + } + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("blank", [" ", "\t", ""], ids=["spaces", "tab", "empty"]) +def test_a_blank_warehouse_falls_back_to_the_bucket(catalog, blank): + resolved = resolve_iceberg_catalog(_binding(catalog, bucket="lake", warehouse=blank)) + assert resolved.warehouse == "s3://lake/streaming/orders/" + dbz = emit_debezium_iceberg_sink_config(resolved) + assert dbz.get("warehouse") == "s3://lake/streaming/orders/", dbz + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_blank_warehouse_and_no_bucket_is_refused(catalog, engine): + contract = _contract(_binding(catalog, warehouse=" "), engine) + errors, _warnings = validate_iceberg_sink(contract) + assert any(f"{catalog} catalog requires binding.location.warehouse" in e for e in errors) + assert iceberg_sink_preflight(contract, "b1") is not None + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_blank_bigquery_project_is_refused(engine): + binding = { + "platform": "gcp", + "format": "iceberg", + "location": { + "database": "streaming", + "table": "orders", + "catalog": "bigquery", + "bucket": "lake", + "project": " ", + }, + } + assert "gcp.bigquery.project-id" not in resolve_iceberg_catalog(binding).extra_catalog_props + contract = _contract(binding, engine) + errors, _warnings = validate_iceberg_sink(contract) + assert any("bigquery catalog requires binding.location.project" in e for e in errors), errors + assert iceberg_sink_preflight(contract, "b1") is not None + + +def test_a_padded_bigquery_project_and_region_are_stripped(): + binding = { + "platform": "gcp", + "format": "iceberg", + "location": { + "database": "streaming", + "table": "orders", + "catalog": "bigquery", + "bucket": "lake", + "project": " acme-proj ", + "region": " EU ", + }, + } + props = resolve_iceberg_catalog(binding).extra_catalog_props + assert props["gcp.bigquery.project-id"] == "acme-proj" + assert props["gcp.bigquery.location"] == "EU" diff --git a/tests/build_runners/test_fw2_debezium_derived_warehouse.py b/tests/build_runners/test_fw2_debezium_derived_warehouse.py new file mode 100644 index 00000000..ac711057 --- /dev/null +++ b/tests/build_runners/test_fw2_debezium_derived_warehouse.py @@ -0,0 +1,164 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An embedded Debezium sink over DynamoDB / JDBC exposes with derived warehouses. + +A warehouse derived from ``location.bucket`` and ``path`` is that expose's +own table prefix, and the catalog creates every missing table under the one +warehouse the sink is given (apache-iceberg-1.10.0 DynamoDbCatalog.java: +163-186, JdbcCatalog.java:282-284). Writing two such exposes through the +first one's prefix put the second one's data inside the first one's prefix, +so differing derived warehouses are refused. +""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.api.runner import RunState +from fluid_build.build_runners._acquisition_common import build_acquisition_run_context +from fluid_build.build_runners.debezium.runner import DebeziumRunner +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) + +pytestmark = [pytest.mark.unit] + +_JDBC_URI = "jdbc:postgresql://catalog-db:5432/iceberg" +_REFUSED = "derive different warehouses from binding.location.bucket and path" + + +def _expose(eid: str, kind: str, **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = { + "database": "sales", + "table": eid, + "bucket": "lake", + "region": "us-east-1", + "catalog": kind, + } + if kind == "jdbc": + loc["uri"] = _JDBC_URI + loc.update(location) + return { + "exposeId": eid, + "kind": "table", + "binding": {"platform": "aws", "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + + +def _contract(exposes: List[Dict[str, Any]], outputs: Optional[List[str]]) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": "ingest", + "pattern": "acquisition", + "engine": "debezium", + "capabilities": ["cdc", "streaming"], + "properties": { + "source": { + "kind": "postgres", + "connection": { + "host": "db", + "port": 5432, + "database": "app", + "user": "u", + "password": "p", + }, + "mode": "cdc", + "streams": [f"public.{e['exposeId']}" for e in exposes], + "watermark": {"strategy": "high_water_mark", "allowedLateness": "PT5M"}, + }, + "sink": {"format": "iceberg"}, + "debezium": {"deployment": {"mode": "embedded"}, "server": {"sink": {}}}, + }, + } + if outputs is not None: + build["outputs"] = outputs + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.cdc", + "name": "Sales CDC", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "x@y.z"}}, + "exposes": copy.deepcopy(exposes), + "builds": [build], + } + + +def _props(contract: Dict[str, Any], tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" + + +def _run(contract: Dict[str, Any], tmp_path: Path): + ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + return DebeziumRunner().run(ctx) + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_a_restricted_expose_is_not_written_under_a_public_prefix(tmp_path: Path, kind): + contract = _contract( + [ + _expose("orders", kind, path="public/orders/"), + _expose("payroll", kind, path="restricted/payroll/"), + ], + ["orders", "payroll"], + ) + errors, _ = validate_iceberg_sink(contract) + refusal = next((e for e in errors if _REFUSED in e), "") + assert "'orders': 's3://lake/public/orders/'" in refusal, errors + assert "'payroll': 's3://lake/restricted/payroll/'" in refusal, errors + assert "Set one binding.location.warehouse on them" in refusal + assert _REFUSED in (iceberg_sink_preflight(contract, "ingest") or "") + result = _run(contract, tmp_path) + assert result.state == RunState.FAILED + assert _REFUSED in (result.error or "") + assert not _props(contract, tmp_path).exists() + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) +def test_default_paths_of_one_database_are_refused(tmp_path: Path, kind, outputs): + # ``path`` defaults to ``/
/``: sales/orders/ and sales/refunds/. + contract = _contract([_expose("orders", kind), _expose("refunds", kind)], outputs) + errors, _ = validate_iceberg_sink(contract) + assert any(_REFUSED in e for e in errors), errors + assert _run(contract, tmp_path).state == RunState.FAILED + assert not _props(contract, tmp_path).exists() + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_one_shared_path_derives_one_warehouse(tmp_path: Path, kind): + contract = _contract( + [_expose("orders", kind, path="sales/"), _expose("refunds", kind, path="sales/")], + None, + ) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None + _run(contract, tmp_path) # no server binary: fails after writing the config + text = _props(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/" in text + assert "debezium.sink.iceberg.table-namespace=sales" in text + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_a_single_expose_keeps_its_derived_warehouse(tmp_path: Path, kind): + contract = _contract([_expose("orders", kind)], ["orders"]) + assert validate_iceberg_sink(contract) == ([], []) + _run(contract, tmp_path) + text = _props(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/orders/" in text diff --git a/tests/build_runners/test_fw2_gcp_dataset_namespace.py b/tests/build_runners/test_fw2_gcp_dataset_namespace.py new file mode 100644 index 00000000..ba001a85 --- /dev/null +++ b/tests/build_runners/test_fw2_gcp_dataset_namespace.py @@ -0,0 +1,152 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A derived sink names its table's namespace from ``location.dataset`` on GCP. + +A GCP binding names its BigQuery dataset in ``location.dataset``. The sink +read only ``location.database``, so it wrote ``iceberg.tables=None.
`` +and ``fluid validate`` passed. A derived sink whose expose names no namespace +is now refused. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + + +def _gcp(table: Optional[str] = "events", **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = {"catalog": "bigquery", "project": "acme-proj", "bucket": "lake"} + if table is not None: + loc["table"] = table + loc.update(location) + return {"platform": "gcp", "format": "iceberg", "location": loc} + + +def _contract( + bindings: List[Dict[str, Any]], + engine: str, + *, + overrides: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + if overrides: + props["kafka-connect"]["iceberg_catalog_overrides"] = dict(overrides) + exposes = [ + { + "exposeId": f"e{i}", + "kind": "table", + "binding": copy.deepcopy(b), + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + for i, b in enumerate(bindings) + ] + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": exposes, + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": [e["exposeId"] for e in exposes], + "properties": props, + } + ], + } + + +def _namespace_errors(errors: List[str]) -> List[str]: + return [e for e in errors if "names no table namespace" in e] + + +def test_a_gcp_dataset_names_the_kafka_connect_table(): + resolved = resolve_iceberg_catalog(_gcp(dataset="analytics")) + assert resolved.fq_table == "analytics.events" + cfg = emit_iceberg_sink_config(resolved, product_id="bronze.o", topics=["public.o"]) + assert cfg["iceberg.tables"] == "analytics.events" + + +def test_a_gcp_dataset_names_the_debezium_namespace(): + resolved = resolve_iceberg_catalog(_gcp(dataset="analytics")) + assert emit_debezium_iceberg_sink_config(resolved)["table-namespace"] == "analytics" + + +def test_location_database_wins_over_dataset(): + resolved = resolve_iceberg_catalog(_gcp(database="raw", dataset="analytics")) + assert resolved.fq_table == "raw.events" + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_gcp_dataset_validates(engine): + errors, _warnings = validate_iceberg_sink(_contract([_gcp(dataset="analytics")], engine)) + assert _namespace_errors(errors) == [], errors + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_derived_sink_with_no_namespace_is_refused(engine): + contract = _contract([_gcp()], engine) + errors, _warnings = validate_iceberg_sink(contract) + assert len(_namespace_errors(errors)) == 1, errors + assert "binding.location.dataset" in _namespace_errors(errors)[0] + assert iceberg_sink_preflight(contract, "b1") is not None + + +def test_a_derived_kafka_connect_sink_with_no_table_is_refused(): + errors, _warnings = validate_iceberg_sink( + _contract([_gcp(table=None, dataset="analytics")], "kafka-connect") + ) + assert any("names no table, so the derived iceberg.tables" in e for e in errors), errors + + +def test_an_override_that_names_the_tables_is_not_refused(): + contract = _contract( + [_gcp()], "kafka-connect", overrides={"iceberg.tables": "analytics.events"} + ) + errors, _warnings = validate_iceberg_sink(contract) + assert _namespace_errors(errors) == [], errors + + +def test_debezium_exposes_sharing_one_dataset_are_one_namespace(): + contract = _contract( + [_gcp(dataset="analytics"), _gcp(table="refunds", dataset="analytics")], + "debezium", + ) + errors, _warnings = validate_iceberg_sink(contract) + assert not [e for e in errors if "databases" in e or "names no table" in e], errors diff --git a/tests/build_runners/test_fw2_integration_seams.py b/tests/build_runners/test_fw2_integration_seams.py new file mode 100644 index 00000000..a814fa1d --- /dev/null +++ b/tests/build_runners/test_fw2_integration_seams.py @@ -0,0 +1,380 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The sink-expose resolver and the catalog-kind derivation, composed. + +The resolver picks the exposes a derived streaming sink writes from the +build's ``outputs``; the catalog table derives what each kind needs to start +(a DynamoDB / JDBC warehouse from the bucket, the BigQuery project and +location). Each was tested on its own. Here a build writes a DynamoDB expose +and another a BigQuery expose in one contract, an embedded Debezium build +writes two exposes of one JDBC or DynamoDB database, and a Confluent Tableflow +expose with no catalog sits beside a self-managed sink with access grants. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest +import yaml + +from fluid_build.api.runner import RunState +from fluid_build.build_runners._acquisition_common import build_acquisition_run_context +from fluid_build.build_runners.debezium.runner import DebeziumRunner +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.build_runners.kafka_connect.runner import execute_kafka_connect_build +from fluid_build.cli import policy_compile +from fluid_build.policy.compiler import compile_policy + +pytestmark = [pytest.mark.unit] + +_SOURCE = { + "kind": "postgres", + "connection": {"host": "db", "port": 5432, "database": "app", "user": "u", "password": "p"}, + "mode": "incremental_append", + "watermark": {"strategy": "high_water_mark", "allowedLateness": "PT5M"}, +} + +_JDBC_URI = "jdbc:postgresql://catalog-db:5432/iceberg" + + +def _expose(eid: str, platform: str, **location: Any) -> Dict[str, Any]: + return { + "exposeId": eid, + "kind": "table", + "binding": {"platform": platform, "format": "iceberg", "location": location}, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + + +def _kc_build(bid: str, outputs: Optional[List[str]], stream: str) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": bid, + "pattern": "acquisition", + "engine": "kafka-connect", + "capabilities": ["streaming"], + "properties": { + "source": {**copy.deepcopy(_SOURCE), "streams": [stream]}, + "sink": {"format": "iceberg"}, + "kafka-connect": { + "deployment": {"server_url": "http://kafka-connect.test:8083"}, + "connector_name": f"src-{bid}", + }, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _dbz_build(outputs: Optional[List[str]]) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": "ingest", + "pattern": "acquisition", + "engine": "debezium", + "capabilities": ["cdc", "streaming"], + "properties": { + "source": { + **copy.deepcopy(_SOURCE), + "mode": "cdc", + "streams": ["public.orders", "public.refunds"], + }, + "sink": {"format": "iceberg"}, + "debezium": {"deployment": {"mode": "embedded"}, "server": {"sink": {}}}, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _contract( + exposes: List[Dict[str, Any]], + builds: List[Dict[str, Any]], + *, + grants: bool = False, +) -> Dict[str, Any]: + doc: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.cdc", + "name": "Sales CDC", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "x@y.z"}}, + "exposes": copy.deepcopy(exposes), + "builds": copy.deepcopy(builds), + } + if grants: + doc["accessPolicy"] = { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + } + return doc + + +# ── a DynamoDB expose and a BigQuery expose, one Kafka Connect build each ──── + +DDB = _expose( + "orders", + "aws", + database="sales", + table="orders", + bucket="lake", + region="us-east-1", + catalog="dynamodb", +) +BQ = _expose( + "events", + "gcp", + database="analytics", + table="events", + bucket="gbkt", + region="europe-west1", + project="acme-proj", + catalog="bigquery", +) + + +def _ddb_and_bq() -> Dict[str, Any]: + return _contract( + [DDB, BQ], + [ + _kc_build("ingest_orders", ["orders"], "public.orders"), + _kc_build("ingest_events", ["events"], "public.events"), + ], + ) + + +def test_ddb_and_bq_builds_validate_against_their_own_catalogs(): + errors, warnings = validate_iceberg_sink(_ddb_and_bq()) + assert errors == [] + # the only warning is the BigQuery build's published-sink advisory + assert len(warnings) == 1, warnings + assert "build 'ingest_events'" in warnings[0] + assert "iceberg.catalog.type=bigquery" in warnings[0] + for bid in ("ingest_orders", "ingest_events"): + assert iceberg_sink_preflight(_ddb_and_bq(), bid) is None + + +def test_ddb_build_pushes_the_dynamodb_catalog_with_its_derived_warehouse( + kafka_connect_mock, tmp_path: Path +): + contract = _ddb_and_bq() + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest_orders-sink"]["config"] + assert sink["iceberg.tables"] == "sales.orders" + assert sink["iceberg.catalog.catalog-impl"] == "org.apache.iceberg.aws.dynamodb.DynamoDbCatalog" + assert sink["iceberg.catalog.warehouse"] == "s3://lake/sales/orders/" + assert sink["iceberg.catalog.client.region"] == "us-east-1" + assert not any(k.startswith("iceberg.catalog.gcp.") for k in sink) + + +def test_bq_build_pushes_the_bigquery_catalog_with_its_project_and_location( + kafka_connect_mock, tmp_path: Path +): + contract = _ddb_and_bq() + assert execute_kafka_connect_build(contract["builds"][1], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest_events-sink"]["config"] + assert sink["iceberg.tables"] == "analytics.events" + assert sink["iceberg.catalog.type"] == "bigquery" + assert sink["iceberg.catalog.warehouse"] == "gs://gbkt" + assert sink["iceberg.catalog.gcp.bigquery.project-id"] == "acme-proj" + assert sink["iceberg.catalog.gcp.bigquery.location"] == "europe-west1" + assert "iceberg.catalog.client.region" not in sink + assert "iceberg.catalog.catalog-impl" not in sink + + +def test_a_build_naming_both_kinds_is_refused_before_any_rest_call( + kafka_connect_mock, tmp_path: Path +): + contract = _contract([DDB, BQ], [_kc_build("ingest", ["orders", "events"], "public.orders")]) + errors, _ = validate_iceberg_sink(contract) + assert any("name 2 of the Iceberg sink exposes" in e for e in errors), errors + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) != 0 + assert kafka_connect_mock.connectors == {} + + +def test_bq_build_without_project_is_refused_even_when_the_first_expose_is_complete(): + no_project = copy.deepcopy(BQ) + del no_project["binding"]["location"]["project"] + contract = _contract( + [DDB, no_project], [_kc_build("ingest_events", ["events"], "public.events")] + ) + errors, _ = validate_iceberg_sink(contract) + assert any("bigquery catalog requires binding.location.project" in e for e in errors), errors + + +# ── embedded Debezium over two exposes of one JDBC / DynamoDB database ────── + + +def _two_tables(kind: str, **extra: Any) -> List[Dict[str, Any]]: + common: Dict[str, Any] = { + "database": "sales", + "bucket": "lake", + "region": "us-east-1", + "catalog": kind, + } + if kind == "jdbc": + common["uri"] = _JDBC_URI + common.update(extra) + return [ + _expose("orders", "aws", table="orders", **common), + _expose("refunds", "aws", table="refunds", **common), + ] + + +def _props_path(contract: Dict[str, Any], tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" + + +_DERIVED = "derive different warehouses from binding.location.bucket and path" + + +@pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) +def test_debezium_refuses_two_bucket_derived_exposes_of_one_database(tmp_path: Path, kind, outputs): + # Each expose derives ``s3://lake/sales/
/``, its own table prefix. + # The catalog creates every missing table under the one warehouse the + # sink is given, so writing both through either prefix is refused. + contract = _contract(_two_tables(kind), [_dbz_build(outputs)]) + errors, _ = validate_iceberg_sink(contract) + assert any(_DERIVED in e for e in errors), errors + assert _DERIVED in (iceberg_sink_preflight(contract, "ingest") or "") + result = DebeziumRunner().run( + build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + ) + assert result.state == RunState.FAILED + assert not _props_path(contract, tmp_path).exists() + + +@pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) +def test_debezium_writes_two_exposes_of_one_database_under_one_warehouse(tmp_path: Path, kind): + contract = _contract(_two_tables(kind, warehouse="s3://lake/sales/"), [_dbz_build(None)]) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None + ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + DebeziumRunner().run(ctx) # no server binary: fails after writing the config + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=sales" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/" in text + + +@pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) +def test_debezium_refuses_bucket_derived_exposes_in_two_buckets(tmp_path: Path, kind): + exposes = _two_tables(kind) + exposes[1]["binding"]["location"]["bucket"] = "other-lake" + contract = _contract(exposes, [_dbz_build(None)]) + errors, _ = validate_iceberg_sink(contract) + assert any(_DERIVED in e for e in errors), errors + result = DebeziumRunner().run( + build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + ) + assert result.state == RunState.FAILED + assert _DERIVED in (result.error or "") + assert not _props_path(contract, tmp_path).exists() + + +def test_debezium_compares_a_warehouse_the_binding_sets_whole(): + exposes = _two_tables("jdbc") + exposes[0]["binding"]["location"]["warehouse"] = "s3://lake/a/" + exposes[1]["binding"]["location"]["warehouse"] = "s3://lake/b/" + errors, _ = validate_iceberg_sink(_contract(exposes, [_dbz_build(None)])) + assert any("their warehouse differ" in e for e in errors), errors + exposes[1]["binding"]["location"]["warehouse"] = "s3://lake/a/" + assert validate_iceberg_sink(_contract(exposes, [_dbz_build(None)])) == ([], []) + + +@pytest.mark.parametrize( + "key, value, named", + [("project", "other-proj", "project"), ("region", "us-central1", "region")], +) +def test_debezium_names_the_bigquery_setting_that_differs(key, value, named): + # ``location.project`` / ``location.region`` reach the catalog as + # gcp.bigquery.project-id / gcp.bigquery.location; the refusal names the + # binding setting. + first = copy.deepcopy(BQ) + first["exposeId"] = "orders" + second = copy.deepcopy(BQ) + second["exposeId"] = "refunds" + second["binding"]["location"]["table"] = "refunds" + second["binding"]["location"][key] = value + errors, _ = validate_iceberg_sink(_contract([first, second], [_dbz_build(None)])) + assert any(f"resolve to different catalogs (their {named} differ)" in e for e in errors), errors + + +# ── a Confluent Tableflow expose with no catalog, beside a sink, with grants ─ + +TABLEFLOW = _expose( + "tableflow_orders", + "confluent", + environment_id="env-123", + kafka_cluster_id="lkc-123", + topic="orders", + database="sales_glue", + bucket="acme-tableflow", + confluent_role_arn="arn:aws:iam::123456789012:role/tableflow", + region="eu-west-1", +) +GLUE = _expose("orders", "aws", database="sales", table="orders", bucket="lake", region="us-east-1") + + +def test_a_tableflow_expose_is_not_a_sink_candidate(): + # one self-managed Iceberg expose: a build with no outputs writes it + contract = _contract([TABLEFLOW, GLUE], [_kc_build("ingest", None, "public.orders")]) + assert validate_iceberg_sink(contract) == ([], []) + + +def test_a_derived_sink_naming_only_the_tableflow_expose_is_refused( + kafka_connect_mock, tmp_path: Path +): + contract = _contract( + [TABLEFLOW, GLUE], [_kc_build("ingest", ["tableflow_orders"], "public.orders")] + ) + errors, _ = validate_iceberg_sink(contract) + assert any( + "outputs ['tableflow_orders'] name none of the Iceberg sink exposes" in e for e in errors + ), errors + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) != 0 + assert kafka_connect_mock.connectors == {} + + +def test_policy_compile_grants_both_glue_tables_and_prints_no_warning(tmp_path: Path, caplog): + contract = _contract( + [TABLEFLOW, GLUE], [_kc_build("ingest", ["orders"], "public.orders")], grants=True + ) + bindings, warnings = compile_policy(contract) + assert warnings == [] + assert [(b["provider"], b["resource_type"], b["resource_id"]) for b in bindings] == [ + ("aws", "s3.bucket", "acme-tableflow"), + ("aws", "glue.table", "sales_glue.orders"), + ("aws", "s3.bucket", "lake"), + ("aws", "glue.table", "sales.orders"), + ] + + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + out = tmp_path / "bindings.json" + logger = logging.getLogger("test.fw2.seams") + caplog.set_level(logging.DEBUG, logger="test.fw2.seams") + assert policy_compile.run(argparse.Namespace(contract=str(path), out=str(out)), logger) == 0 + assert json.loads(out.read_text())["bindings"] == bindings + shown = [r for r in caplog.records if r.levelno >= logging.WARNING] + assert shown == [], [r.getMessage() for r in shown] diff --git a/tests/build_runners/test_fw2_sink_expose.py b/tests/build_runners/test_fw2_sink_expose.py new file mode 100644 index 00000000..3595b167 --- /dev/null +++ b/tests/build_runners/test_fw2_sink_expose.py @@ -0,0 +1,363 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LOGIC-1: a derived streaming Iceberg sink writes the exposes its build's outputs pick. + +Two CDC builds feeding two Iceberg exposes is the ordinary shape. Every sink +used to write ``exposes[0]``, so the ``refunds`` build's connector carried +``iceberg.tables=sales.orders`` while ``fluid validate``, the preflight and the +run all passed. A sink config forge-cli derives now resolves its exposes from +``build.outputs`` through one resolver that ``fluid validate``, the preflight, +both runners and the late-arrival target read; a join it cannot resolve is a +HARD error that stops the run before any Connect REST call or config write. A +hand-written config, or an override that names the tables itself, behaves as +before. +""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.api.runner import RunState +from fluid_build.build_runners._acquisition_common import build_acquisition_run_context +from fluid_build.build_runners.debezium.runner import DebeziumRunner +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.build_runners.kafka_connect.runner import execute_kafka_connect_build + +pytestmark = [pytest.mark.unit] + +_SOURCE = { + "kind": "postgres", + "connection": {"host": "db", "port": 5432, "database": "app", "user": "u", "password": "p"}, + "mode": "incremental_append", + "watermark": {"strategy": "high_water_mark", "allowedLateness": "PT5M"}, +} + +_LAKEKEEPER = {"catalog": "lakekeeper", "uri": "http://lakekeeper:8181/catalog", "warehouse": "wh"} + + +def _expose( + eid: str, database: str, table: str, *, fmt: str = "iceberg", **location: Any +) -> Dict[str, Any]: + return { + "exposeId": eid, + "kind": "table", + "binding": { + "platform": "aws", + "format": fmt, + "location": { + "database": database, + "table": table, + "bucket": "lake", + "region": "us-east-1", + **location, + }, + }, + "contract": {"schema": []}, + } + + +def _kc_build(bid: str, outputs: Optional[List[str]], stream: str, **kc: Any) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": bid, + "pattern": "acquisition", + "engine": "kafka-connect", + "capabilities": ["streaming"], + "properties": { + "source": {**copy.deepcopy(_SOURCE), "streams": [stream]}, + "sink": {"format": "iceberg"}, + "kafka-connect": { + "deployment": {"server_url": "http://kafka-connect.test:8083"}, + "connector_name": f"src-{bid}", + **kc, + }, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _dbz_build(outputs: Optional[List[str]], **server_sink: Any) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": "ingest", + "pattern": "acquisition", + "engine": "debezium", + "capabilities": ["cdc", "streaming"], + "properties": { + "source": { + **copy.deepcopy(_SOURCE), + "mode": "cdc", + "streams": ["public.orders", "public.refunds"], + }, + "sink": {"format": "iceberg"}, + "debezium": {"deployment": {"mode": "embedded"}, "server": {"sink": server_sink}}, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _contract(exposes: List[Dict[str, Any]], builds: List[Dict[str, Any]]) -> Dict[str, Any]: + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.cdc", + "name": "Sales CDC", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "x@y.z"}}, + # copies: a test edits its contract, and the module-level exposes are shared + "exposes": copy.deepcopy(exposes), + "builds": copy.deepcopy(builds), + } + + +ORDERS = _expose("orders", "sales", "orders") +REFUNDS = _expose("refunds", "sales", "refunds") +FIN_REFUNDS = _expose("refunds", "finance", "refunds") +RAW = _expose("raw", "sales", "raw", fmt="parquet") + + +def _two_builds() -> Dict[str, Any]: + return _contract( + [ORDERS, FIN_REFUNDS], + [ + _kc_build("ingest_orders", ["orders"], "public.orders"), + _kc_build("ingest_refunds", ["refunds"], "public.refunds"), + ], + ) + + +def _run_embedded(contract: Dict[str, Any], tmp_path: Path): + ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + return DebeziumRunner().run(ctx) + + +def _props_path(contract: Dict[str, Any], tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" + + +# ── the defect: each derived sink writes ITS build's expose ───────────────── + + +def test_two_builds_validate_clean(): + assert validate_iceberg_sink(_two_builds()) == ([], []) + + +@pytest.mark.parametrize( + "index, table", + [(0, "sales.orders"), (1, "finance.refunds")], + ids=["first-build", "second-build"], +) +def test_kafka_connect_sink_and_late_arrival_target_follow_the_builds_expose( + kafka_connect_mock, tmp_path: Path, index, table +): + contract = _two_builds() + build = contract["builds"][index] + assert execute_kafka_connect_build(build, contract, tmp_path, dry_run=False) == 0 + name = build["properties"]["kafka-connect"]["connector_name"] + sink = kafka_connect_mock.connectors[f"{name}-sink"]["config"] + assert sink["iceberg.tables"] == table + source = kafka_connect_mock.connectors[name]["config"] + # the late-events side table sits beside the same target + assert source["fluid.late_arrival.side_output_table"] == f"{table}__late_events" + + +def test_embedded_debezium_namespace_follows_the_builds_expose(tmp_path: Path): + contract = _contract([ORDERS, FIN_REFUNDS], [_dbz_build(["refunds"])]) + assert validate_iceberg_sink(contract) == ([], []) + _run_embedded(contract, tmp_path) # no server binary: fails after writing the config + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=finance" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/finance/refunds/" in text + assert "table-namespace=sales" not in text + + +def test_validate_checks_the_catalog_of_the_expose_the_build_writes(): + # outputs pick the Lakekeeper expose, which has no uri; the first expose + # is a complete Glue binding, which is all the validator used to read. + no_uri = _expose("refunds", "sales", "refunds", catalog="lakekeeper", warehouse="wh") + contract = _contract([ORDERS, no_uri], [_kc_build("ingest", ["refunds"], "public.refunds")]) + errors, _ = validate_iceberg_sink(contract) + assert any("lakekeeper catalog requires binding.location.uri" in e for e in errors), errors + + +# ── Kafka Connect: a derived sink writes one expose; anything else is HARD ── + +_KC_REFUSED = [ + ([ORDERS, RAW], ["raw"], "outputs ['raw'] name none of the Iceberg sink exposes"), + ([ORDERS, REFUNDS], ["orders", "refunds"], "outputs ['orders', 'refunds'] name 2 of"), + ([ORDERS, REFUNDS], None, "declares no outputs, and the contract has 2 Iceberg sink exposes"), + ([ORDERS, REFUNDS], [], "declares no outputs, and the contract has 2 Iceberg sink exposes"), +] +_KC_IDS = ["outputs-name-non-iceberg", "outputs-name-two", "no-outputs-two", "empty-outputs-two"] + + +@pytest.mark.parametrize("exposes, outputs, message", _KC_REFUSED, ids=_KC_IDS) +def test_kafka_connect_validate_and_preflight_refuse(exposes, outputs, message): + contract = _contract(exposes, [_kc_build("ingest", outputs, "public.orders")]) + errors, warnings = validate_iceberg_sink(contract) + [error] = [e for e in errors if message in e] + # names the build, its candidate exposes, and the way out + assert "build 'ingest'" in error + assert "orders (sales.orders)" in error + assert "a derived Kafka Connect sink writes one expose" in error + assert "split the build" in error and "hand-write the sink config" in error + assert not any("join is implicit" in w for w in warnings) + preflight = iceberg_sink_preflight(contract, "ingest") + assert preflight is not None and message in preflight + + +@pytest.mark.parametrize("exposes, outputs, message", _KC_REFUSED, ids=_KC_IDS) +def test_kafka_connect_run_refuses_before_any_rest_call( + kafka_connect_mock, tmp_path: Path, exposes, outputs, message +): + contract = _contract(exposes, [_kc_build("ingest", outputs, "public.orders")]) + rc = execute_kafka_connect_build(contract["builds"][0], contract, tmp_path, dry_run=False) + assert rc != 0 + assert kafka_connect_mock.connectors == {} + + +# ── embedded Debezium: one namespace and one catalog ──────────────────────── + + +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) +def test_embedded_debezium_writes_one_namespace_for_several_exposes(tmp_path: Path, outputs): + # DefaultIcebergTableMapper.mapDestination writes each captured table + # under table-namespace, so two exposes in one database and one catalog + # are one derived sink. + contract = _contract([ORDERS, REFUNDS], [_dbz_build(outputs)]) + assert validate_iceberg_sink(contract) == ([], []) + _run_embedded(contract, tmp_path) # no server binary: fails after writing the config + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=sales" in text + + +_DBZ_REFUSED = [ + ([ORDERS, RAW], ["raw"], "outputs ['raw'] name none of the Iceberg sink exposes"), + ([ORDERS, FIN_REFUNDS], ["orders", "refunds"], "sit in the databases ['finance', 'sales']"), + ([ORDERS, FIN_REFUNDS], None, "sit in the databases ['finance', 'sales']"), + ( + [ORDERS, _expose("refunds", "sales", "refunds", **_LAKEKEEPER)], + ["orders", "refunds"], + "resolve to different catalogs (their catalog, warehouse, uri differ)", + ), + ( + [ORDERS, _expose("refunds", "sales", "refunds", region="eu-west-1")], + None, + "resolve to different catalogs (their region differ)", + ), +] +_DBZ_IDS = [ + "outputs-name-non-iceberg", + "two-databases", + "no-outputs-two-databases", + "two-catalog-kinds", + "no-outputs-two-regions", +] + + +@pytest.mark.parametrize("exposes, outputs, message", _DBZ_REFUSED, ids=_DBZ_IDS) +def test_embedded_debezium_validate_preflight_and_run_refuse( + tmp_path: Path, exposes, outputs, message +): + contract = _contract(exposes, [_dbz_build(outputs)]) + errors, _ = validate_iceberg_sink(contract) + assert any("build 'ingest'" in e and message in e for e in errors), errors + preflight = iceberg_sink_preflight(contract, "ingest") + assert preflight is not None and message in preflight + result = _run_embedded(contract, tmp_path) + assert result.state == RunState.FAILED + assert message in (result.error or "") + assert not _props_path(contract, tmp_path).exists() + + +# ── unchanged: single-expose contracts and configs that name their tables ─── + + +@pytest.mark.parametrize("outputs", [None, [], ["orders"]], ids=["absent", "empty", "named"]) +def test_single_expose_kafka_connect_contract_is_unchanged( + kafka_connect_mock, tmp_path: Path, outputs +): + contract = _contract([ORDERS], [_kc_build("ingest", outputs, "public.orders")]) + assert validate_iceberg_sink(contract) == ([], []) + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest-sink"]["config"] + assert sink["iceberg.tables"] == "sales.orders" + + +@pytest.mark.parametrize("outputs", [None, ["orders"]], ids=["absent", "named"]) +def test_single_expose_embedded_debezium_contract_is_unchanged(tmp_path: Path, outputs): + contract = _contract([ORDERS], [_dbz_build(outputs)]) + assert validate_iceberg_sink(contract) == ([], []) + _run_embedded(contract, tmp_path) + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=sales" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/orders/" in text + + +@pytest.mark.parametrize( + "kc", + [ + {"iceberg_catalog_overrides": {"iceberg.tables": "sales.refunds"}}, + { + "iceberg_sink_enabled": True, + "sink_connector_config": {"iceberg.tables": "sales.refunds"}, + }, + {"sink_connector_config": {"iceberg.tables": "sales.refunds"}}, + ], + ids=["override", "hand-written-over-derived", "hand-written"], +) +@pytest.mark.parametrize("outputs", [None, ["orders", "refunds"]], ids=["absent", "two"]) +def test_kafka_connect_config_naming_its_tables_is_unchanged( + kafka_connect_mock, tmp_path: Path, kc, outputs +): + contract = _contract([ORDERS, REFUNDS], [_kc_build("ingest", outputs, "public.refunds", **kc)]) + assert validate_iceberg_sink(contract) == ([], []) + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest-sink"]["config"] + assert sink["iceberg.tables"] == "sales.refunds" + # the late-arrival target is still the first Iceberg expose, as before + source = kafka_connect_mock.connectors["src-ingest"]["config"] + assert source["fluid.late_arrival.side_output_table"] == "sales.orders__late_events" + + +def test_kafka_connect_hand_written_outputs_naming_no_expose_still_warn(): + kc = {"sink_connector_config": {"iceberg.tables": "sales.orders"}} + contract = _contract([ORDERS, RAW], [_kc_build("ingest", ["raw"], "public.orders", **kc)]) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert any("join is implicit" in w for w in warnings) + + +@pytest.mark.parametrize( + "server_sink", + [ + {"iceberg_sink_enabled": True, "config": {"table-namespace": "finance"}}, + {"config": {"catalog-impl": "org.apache.iceberg.aws.glue.GlueCatalog"}}, + ], + ids=["namespace-override", "hand-written"], +) +def test_embedded_debezium_config_naming_its_namespace_is_unchanged(tmp_path: Path, server_sink): + # two databases would refuse a derived namespace; these name their own + contract = _contract([ORDERS, FIN_REFUNDS], [_dbz_build(None, **server_sink)]) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None diff --git a/tests/build_runners/test_iceberg_sink_validation.py b/tests/build_runners/test_iceberg_sink_validation.py index 48b8b504..46d97bb3 100644 --- a/tests/build_runners/test_iceberg_sink_validation.py +++ b/tests/build_runners/test_iceberg_sink_validation.py @@ -90,9 +90,21 @@ def test_iceberg_sink_without_iceberg_expose_errors(): assert any("no expose with binding.format=iceberg" in e for e in errs) -def test_outputs_not_referencing_iceberg_expose_warns(): - warns = _warns(_contract(outputs=["something_else"])) - assert any("don't reference the Iceberg expose" in w for w in warns) +def test_outputs_not_referencing_iceberg_expose_errors_for_a_derived_sink(): + # A derived sink writes the expose its outputs name, so outputs that name + # none leave it no table; this was a warning, and the sink wrote exposes[0]. + contract = _contract(outputs=["something_else"]) + assert any( + "name none of the Iceberg sink exposes ['events (s.o)']" in e for e in _errs(contract) + ) + assert _warns(contract) == [] + + +def test_outputs_not_referencing_iceberg_expose_warns_for_a_hand_written_sink(): + hand_written = {"sink_connector_config": {"iceberg.tables": "s.o"}} + contract = _contract(outputs=["something_else"], kc=hand_written) + assert _errs(contract) == [] + assert any("don't reference the Iceberg expose" in w for w in _warns(contract)) def test_iceberg_table_alias_binding_counts_as_iceberg_expose(): @@ -695,7 +707,8 @@ def test_gcp_sink_catalog_bigquery_agrees_with_biglake_and_warns_the_runtime(): # "disagrees with 'rest'" error (dbt and the IaC never read 'rest' here). # The published sink still cannot load type=bigquery, which is a warning. contract = _sink_contract( - _loc_binding(platform="gcp", warehouse="gs://acme/wh"), sink={"catalog": "bigquery"} + _loc_binding(platform="gcp", warehouse="gs://acme/wh", project="acme-proj"), + sink={"catalog": "bigquery"}, ) errors, warnings = validate_iceberg_sink(contract) assert errors == [] @@ -735,7 +748,7 @@ def test_bigquery_warning_follows_the_type_that_reaches_the_worker(): @pytest.mark.parametrize( "catalog, location", [ - ("bigquery", {"warehouse": "gs://acme/wh"}), + ("bigquery", {"warehouse": "gs://acme/wh", "project": "acme-proj"}), ("rest", _GCP_REST_LOC), ("lakekeeper", COMPLETE), ], @@ -958,7 +971,9 @@ def test_unnamed_catalog_off_gcp_is_not_refused(): def test_bigquery_on_kafka_connect_warns_about_the_published_sink(): - contract = _sink_contract(_loc_binding("bigquery", platform="gcp", warehouse="gs://acme/wh")) + contract = _sink_contract( + _loc_binding("bigquery", platform="gcp", warehouse="gs://acme/wh", project="acme-proj") + ) errors, warnings = validate_iceberg_sink(contract) assert errors == [] hit = [w for w in warnings if "iceberg.catalog.type=bigquery" in w] diff --git a/tests/cli/test_fw2_policy_compile_read_blocks.py b/tests/cli/test_fw2_policy_compile_read_blocks.py new file mode 100644 index 00000000..e2dd1308 --- /dev/null +++ b/tests/cli/test_fw2_policy_compile_read_blocks.py @@ -0,0 +1,227 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A ``policy compile`` crash names the value that crashed it, and a contract +with no grants is not a warning. + +A crash inside ``compile_policy`` is reported with the schema type errors +``fluid validate`` finds at the values the compiler reads (the grants, their +permissions, each expose's binding and location). A schema error elsewhere, +such as an extra key on a grant, did not make the compiler fail and is not +named. Once compile warnings reached the console, a contract with no grants (a +legitimate no-op) must not print one on compile, on apply, or in stage 3. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import logging +from pathlib import Path +from typing import Any, Dict, List + +import pytest +import yaml + +from fluid_build.cli import policy_apply, policy_compile +from fluid_build.cli._common import CLIError + +pytestmark = pytest.mark.unit + +_LOGGER = "test.fw2.policy.blocks" + +_CONTRACT: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "analytics.pol", + "name": "Pol", + "metadata": {"owner": {"team": "data"}, "layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": {"bucket": "acme-lake", "path": "orders/", "region": "eu-west-1"}, + }, + "contract": {"schema": [{"name": "order_id", "type": "integer", "required": True}]}, + } + ], +} + +_BINDING_ERROR = "exposes[0].binding is not of type 'object'" + + +def _write(tmp_path: Path, doc: Dict[str, Any]) -> Path: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + return path + + +def _compile_error(tmp_path: Path, doc: Dict[str, Any]) -> str: + args = argparse.Namespace(contract=str(_write(tmp_path, doc)), out=str(tmp_path / "b.json")) + with pytest.raises(CLIError) as excinfo: + policy_compile.run(args, logging.getLogger(_LOGGER)) + assert excinfo.value.event == "policy_compiler_crashed" + return str(excinfo.value.context["error"]) + + +def _warning_events(caplog) -> List[Dict[str, Any]]: + out = [] + for record in caplog.records: + if record.levelno < logging.WARNING: + continue + try: + out.append(json.loads(record.getMessage())) + except ValueError: + out.append({"message": record.getMessage()}) + return out + + +def test_a_string_binding_is_reported_as_its_schema_error(tmp_path): + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"] = "nope" + + error = _compile_error(tmp_path, doc) + + assert _BINDING_ERROR in error + assert "AttributeError" not in error + + +def test_an_unrelated_grant_error_is_not_named(tmp_path): + """The extra ``note`` key does not crash the compiler; the binding does. + Only the binding is named.""" + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"] = "nope" + doc["accessPolicy"]["grants"][0]["note"] = "x" + + error = _compile_error(tmp_path, doc) + + assert error == f"policy compile failed on contract values of the wrong type: {_BINDING_ERROR}" + assert "note" not in error + + +@pytest.mark.parametrize( + "mutate, expected", + [ + ( + lambda d: d.update(accessPolicy=["x"]), + "accessPolicy is not of type 'object'", + ), + ( + lambda d: d["accessPolicy"].update(grants="g"), + "accessPolicy.grants is not of type 'array'", + ), + ( + lambda d: d["accessPolicy"]["grants"][0].update(permissions=None), + "accessPolicy.grants[0].permissions is not of type 'array'", + ), + ( + lambda d: d["exposes"][0]["binding"].update(location=None), + "exposes[0].binding.location is not of type 'object'", + ), + ], + ids=["accessPolicy", "grants", "permissions", "location"], +) +def test_each_value_the_compiler_reads_is_named_when_it_crashes(tmp_path, mutate, expected): + """Each of these crashes ``compile_policy`` today; the error names the value.""" + doc = copy.deepcopy(_CONTRACT) + mutate(doc) + + assert _compile_error(tmp_path, doc) == ( + f"policy compile failed on contract values of the wrong type: {expected}" + ) + + +def test_a_schema_error_outside_the_blocks_the_compiler_reads_is_not_named(tmp_path, monkeypatch): + def _boom(contract): + raise KeyError("location") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + doc = copy.deepcopy(_CONTRACT) + doc["metadata"]["layer"] = 7 + + assert _compile_error(tmp_path, doc) == "policy compiler failed: KeyError: 'location'" + + +def test_stage_3_does_not_warn_for_a_contract_with_no_grants(tmp_path, caplog): + """``fluid generate artifacts`` compiles the policy through the same run().""" + from fluid_build.forge.core.artifact_fanout import _emit_policies + + caplog.set_level(logging.DEBUG) + doc = copy.deepcopy(_CONTRACT) + del doc["accessPolicy"] + + _emit_policies(_write(tmp_path, doc), tmp_path / "policy", logging.getLogger(_LOGGER)) + + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + + +def test_a_contract_with_no_grants_compiles_and_applies_without_a_warning( + tmp_path, monkeypatch, caplog +): + from fluid_build.policy.compiler import NO_GRANTS + + caplog.set_level(logging.DEBUG, logger=_LOGGER) + doc = copy.deepcopy(_CONTRACT) + del doc["accessPolicy"] + out = tmp_path / "policy" / "bindings.json" + args = argparse.Namespace(contract=str(_write(tmp_path, doc)), out=str(out)) + + assert policy_compile.run(args, logging.getLogger(_LOGGER)) == 0 + # The file still says why it is empty. + assert json.loads(out.read_text())["warnings"] == [NO_GRANTS] + + monkeypatch.chdir(tmp_path) + apply_args = argparse.Namespace( + bindings="policy/bindings.json", mode="check", provider=None, project=None + ) + assert policy_apply.run(apply_args, logging.getLogger(_LOGGER)) == 0 + + assert _warning_events(caplog) == [] + + +def test_grants_that_compile_to_nothing_still_warn(tmp_path, caplog): + """The control: a grant over an expose the compiler cannot grant on.""" + caplog.set_level(logging.DEBUG, logger=_LOGGER) + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"] = { + "platform": "azure", + "format": "delta", + "location": {"container": "lake"}, + } + args = argparse.Namespace(contract=str(_write(tmp_path, doc)), out=str(tmp_path / "b.json")) + + assert policy_compile.run(args, logging.getLogger(_LOGGER)) == 0 + + shown = [e.get("warning") for e in _warning_events(caplog)] + assert "No IAM bindings generated from contract" in shown + + +def test_the_crash_error_names_the_path_and_never_the_value(tmp_path): + """A wrong-typed value can be a connection URL with a password in it.""" + secret = "Sup3rS3cretPw" # pragma: allowlist secret + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"]["location"] = f"snowflake://svc_user:{secret}@xy12345/DB/S/T" + + error = _compile_error(tmp_path, doc) + + assert "exposes[0].binding.location is not of type 'object'" in error + assert secret not in error + assert "snowflake://" not in error diff --git a/tests/cli/test_fw2_policy_compile_schema_error.py b/tests/cli/test_fw2_policy_compile_schema_error.py new file mode 100644 index 00000000..442152ca --- /dev/null +++ b/tests/cli/test_fw2_policy_compile_schema_error.py @@ -0,0 +1,103 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A ``policy compile`` crash names the block at fault and the command to run. + +``policy compile`` does not schema-validate the contract, so a grant written as +a bare string reaches ``compile_policy`` and fails there with +``AttributeError: 'str' object has no attribute 'get'``. Once that crash +became the command's failure (RT-707-3), it must name the value at fault and +point at ``fluid validate``, not at the agent-policy block or the sovereignty +page that ``policy_compile_failed`` routes to. +""" + +from __future__ import annotations + +import argparse +import copy +import logging +from pathlib import Path +from typing import Any, Dict + +import pytest +import yaml + +from fluid_build._errors import _DOC_BASE +from fluid_build.cli import policy_compile +from fluid_build.cli._common import CLIError + +pytestmark = pytest.mark.unit + +_CONTRACT: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "analytics.pol", + "name": "Pol", + "metadata": {"owner": {"team": "data"}, "layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": {"bucket": "acme-lake", "path": "orders/", "region": "eu-west-1"}, + }, + "contract": {"schema": [{"name": "order_id", "type": "integer", "required": True}]}, + } + ], +} + + +def _run(tmp_path: Path, doc: Dict[str, Any]) -> CLIError: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + args = argparse.Namespace(contract=str(path), out=str(tmp_path / "o" / "b.json")) + with pytest.raises(CLIError) as excinfo: + policy_compile.run(args, logging.getLogger("test.fw2.policy")) + return excinfo.value + + +def test_a_malformed_grant_is_reported_as_the_schema_error(tmp_path): + doc = copy.deepcopy(_CONTRACT) + doc["accessPolicy"]["grants"] = ["group:analysts@example.com"] + + err = _run(tmp_path, doc) + + assert err.event == "policy_compiler_crashed" + assert err.context["error"] == ( + "policy compile failed on contract values of the wrong type: " + "accessPolicy.grants[0] is not of type 'object'" + ) + assert "AttributeError" not in err.context["error"] + assert not (tmp_path / "o" / "b.json").exists() + + +def test_the_suggestion_points_at_fluid_validate_not_the_agent_policy_block(tmp_path, monkeypatch): + """A crash the schema does not explain keeps the compiler's own error.""" + + def _boom(contract): + raise KeyError("location") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + err = _run(tmp_path, _CONTRACT) + + assert err.context["error"] == "policy compiler failed: KeyError: 'location'" + assert "fluid validate " in err.suggestions[0] + assert "accessPolicy" in err.suggestions[0] + assert "agent-policy" not in " ".join(err.suggestions) + assert err.docs_url == f"{_DOC_BASE}/cli/policy-compile.html#errors" diff --git a/tests/cli/test_fw2_policy_compile_warnings.py b/tests/cli/test_fw2_policy_compile_warnings.py new file mode 100644 index 00000000..9494f869 --- /dev/null +++ b/tests/cli/test_fw2_policy_compile_warnings.py @@ -0,0 +1,204 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid policy compile`` shows its warnings, and fails on a compiler crash. + +RT-707-3: a grant on an Iceberg table in a catalog the compiler cannot grant +on (``catalog: lakekeeper`` on AWS) compiles to no table grant and a warning. +The warning went only into ``bindings.json``; the console got an INFO count +that is routed to DEBUG, so the command printed nothing and exited 0, and +``fluid policy apply`` never read the warnings either. A crash inside +``compile_policy`` was swallowed into a bindings file and exit 0 as well. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, List + +import pytest +import yaml + +from fluid_build.cli import policy_apply, policy_compile +from fluid_build.cli._common import CLIError + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[2] + +_LAKEKEEPER_GRANT: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "analytics.pol", + "name": "Pol", + "metadata": {"owner": {"team": "data"}, "layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "catalog": "lakekeeper", + "uri": "http://lakekeeper:8181/catalog", + "warehouse": "analytics", + "database": "streaming", + "table": "orders", + "bucket": "acme-lake", + "region": "eu-west-1", + }, + }, + "contract": {"schema": [{"name": "order_id", "type": "integer", "required": True}]}, + } + ], +} + +_NO_TABLE_GRANT = "so no table grant was compiled for group:analysts@example.com" + + +def _write_contract(tmp_path: Path, doc: Dict[str, Any]) -> Path: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + return path + + +def _warning_events(caplog, name: str) -> List[Dict[str, Any]]: + out = [] + for record in caplog.records: + if record.levelno != logging.WARNING: + continue + try: + doc = json.loads(record.getMessage()) + except ValueError: + continue + if doc.get("message") == name: + out.append(doc) + return out + + +def test_compile_shows_each_warning_at_warning_level_and_exits_0(tmp_path, caplog): + caplog.set_level(logging.DEBUG, logger="test.fw2.policy") + out = tmp_path / "policy" / "bindings.json" + args = argparse.Namespace( + contract=str(_write_contract(tmp_path, _LAKEKEEPER_GRANT)), out=str(out) + ) + + assert policy_compile.run(args, logging.getLogger("test.fw2.policy")) == 0 + + stored = json.loads(out.read_text())["warnings"] + shown = _warning_events(caplog, "policy_compile_warning") + assert [e["warning"] for e in shown] == stored + assert any(_NO_TABLE_GRANT in w for w in stored) + + +def test_a_compiler_crash_fails_and_writes_no_bindings(tmp_path, monkeypatch): + def _boom(contract): + raise KeyError("location") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + out = tmp_path / "bindings.json" + args = argparse.Namespace( + contract=str(_write_contract(tmp_path, _LAKEKEEPER_GRANT)), out=str(out) + ) + + with pytest.raises(CLIError) as excinfo: + policy_compile.run(args, logging.getLogger("test.fw2.policy")) + + assert excinfo.value.exit_code == 1 + assert excinfo.value.event == "policy_compiler_crashed" + assert "policy compiler failed: KeyError" in excinfo.value.context["error"] + assert not out.exists() + + +def test_a_compiler_crash_leaves_an_earlier_bindings_file_as_it_was(tmp_path, monkeypatch): + out = tmp_path / "bindings.json" + out.write_text('{"bindings": [], "warnings": ["earlier"]}', encoding="utf-8") + + def _boom(contract): + raise RuntimeError("compiler bug") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + args = argparse.Namespace( + contract=str(_write_contract(tmp_path, _LAKEKEEPER_GRANT)), out=str(out) + ) + + with pytest.raises(CLIError): + policy_compile.run(args, logging.getLogger("test.fw2.policy")) + assert out.read_text(encoding="utf-8") == '{"bindings": [], "warnings": ["earlier"]}' + + +def test_apply_echoes_the_stored_warnings(tmp_path, monkeypatch, caplog): + """An empty bindings list is the no-op path, which resolves no provider; + the warnings are echoed before it.""" + monkeypatch.chdir(tmp_path) + caplog.set_level(logging.DEBUG, logger="test.fw2.policy") + warning = "Iceberg expose 'orders' is cataloged in 'lakekeeper', not AWS Glue, " + ( + _NO_TABLE_GRANT + ) + path = tmp_path / "bindings.json" + path.write_text(json.dumps({"bindings": [], "warnings": [warning]}), encoding="utf-8") + args = argparse.Namespace(bindings="bindings.json", mode="check", provider=None, project=None) + + assert policy_apply.run(args, logging.getLogger("test.fw2.policy")) == 0 + + shown = _warning_events(caplog, "policy_bindings_warning") + assert [e["warning"] for e in shown] == [warning] + + +def _cli(tmp_path: Path, *argv: str) -> subprocess.CompletedProcess: + env = {**os.environ, "PYTHONPATH": str(_REPO), "NO_COLOR": "1"} + return subprocess.run( + [sys.executable, "-m", "fluid_build", *argv], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + + +@pytest.mark.integration +def test_the_cli_prints_the_warning_on_stderr(tmp_path): + """The real entry point, as the defect was measured: the warning reaches + stderr (stdout stays the command's own output) and the exit stays 0.""" + contract = _write_contract(tmp_path, _LAKEKEEPER_GRANT) + result = _cli(tmp_path, "policy", "compile", contract.name, "--out", "out/b.json") + + assert result.returncode == 0, result.stderr + assert _NO_TABLE_GRANT in result.stderr + assert _NO_TABLE_GRANT not in result.stdout + + +@pytest.mark.integration +def test_the_cli_fails_on_a_compiler_crash(tmp_path): + """A grant that is a bare string crashes ``compile_policy``; that used to + write ``{"bindings": {"bindings": []}}`` and exit 0.""" + doc = json.loads(json.dumps(_LAKEKEEPER_GRANT)) + doc["accessPolicy"]["grants"] = ["group:analysts@example.com"] + contract = _write_contract(tmp_path, doc) + result = _cli(tmp_path, "policy", "compile", contract.name, "--out", "out/b.json") + + assert result.returncode == 1 + assert "ERR_POLICY_COMPILER_CRASHED" in result.stdout + result.stderr + assert not (tmp_path / "out" / "b.json").exists() diff --git a/tests/iac/test_fw2_snowflake_volume_message.py b/tests/iac/test_fw2_snowflake_volume_message.py new file mode 100644 index 00000000..4f1ae0aa --- /dev/null +++ b/tests/iac/test_fw2_snowflake_volume_message.py @@ -0,0 +1,151 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Snowflake EXTERNAL VOLUME guard names both causes, and blames neither. + +JRN-707-6: the volume is keyed by the contract id, so state cannot say which +expose it was created for. A contract applied only by the current release, with +a Snowflake-managed expose (which gets the volume) beside a Lakekeeper one, +leaves the same volume in state when the managed expose is removed. The guard +still blocks that apply (what it detects is unchanged), but its message said +"forge-cli no longer creates an EXTERNAL VOLUME ..." and listed the Lakekeeper +expose as the table the volume was held for. It now names both possible causes +and keeps the ``tofu state rm`` remediation. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.iac.catalog_moves import ( + CatalogMoveError, + detect_catalog_moves, + guard_catalog_moves, +) +from fluid_build.iac.providers.aws import AwsIacPlugin +from fluid_build.iac.providers.snowflake import SnowflakeIacPlugin + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +SCHEMA = [{"name": "order_id", "type": "string"}] +VOLUME = "snowflake_external_volume.analytics_lake_vol_FLUID_ANALYTICS_LAKE_VOL" +SF_TABLE = "snowflake_table.analytics_lake_ANALYTICS_SALES_ORDERS" +WORKDIR = "/w/snowflake/analytics.lake" + + +def _sf_expose(catalog: Optional[str], expose_id: str, table: str) -> Dict[str, Any]: + location: Dict[str, Any] = { + "database": "ANALYTICS", + "schema": "SALES", + "table": table, + "warehouse": "s3://lake/warehouse", + "iam_role_arn": "arn:aws:iam::123456789012:role/snowflake-lake", + } + if catalog is not None: + location.update(catalog=catalog, uri="http://lakekeeper:8181/catalog") + return { + "exposeId": expose_id, + "binding": {"platform": "snowflake", "format": "iceberg", "location": location}, + "contract": {"schema": copy.deepcopy(SCHEMA)}, + } + + +def _contract(*exposes: Dict[str, Any]) -> Dict[str, Any]: + return {"id": "analytics.lake", "name": "Lake", "exposes": list(exposes)} + + +#: After the change: the managed expose is gone, the Lakekeeper one stays. +_AFTER = _contract(_sf_expose("lakekeeper", "orders_iceberg", "ORDERS")) + + +def _blocked(contract: Dict[str, Any], state) -> CatalogMoveError: + with pytest.raises(CatalogMoveError) as excinfo: + guard_catalog_moves(SnowflakeIacPlugin(), contract, state, workdir=WORKDIR) + return excinfo.value + + +def test_the_current_release_s_own_volume_is_still_blocked(): + """Detection is unchanged: the volume a removed managed expose used is + flagged, as it was before the message changed.""" + plugin = SnowflakeIacPlugin() + before = _contract( + _sf_expose(None, "orders_managed", "ORDERS_MANAGED"), + _sf_expose("lakekeeper", "orders_iceberg", "ORDERS"), + ) + emitted = plugin.emit(before)["snowflake_external_volume"] + assert {f"snowflake_external_volume.{key}" for key in emitted} == {VOLUME} + + assert detect_catalog_moves(plugin, _AFTER, [SF_TABLE, VOLUME]) == (VOLUME,) + exc = _blocked(_AFTER, [SF_TABLE, VOLUME]) + assert exc.addresses == (VOLUME,) + assert exc.remediation == (f"tofu -chdir={WORKDIR} state rm {VOLUME}",) + + +def test_the_message_does_not_say_an_earlier_release_created_it(): + message = str(_blocked(_AFTER, [SF_TABLE, VOLUME])) + + assert "no longer creates" not in message + assert "an earlier dbt run" not in message + assert "for Iceberg table(s) that now live in another catalog" not in message + assert ( + "holds 1 Snowflake EXTERNAL VOLUME(s) that this contract's configuration no longer " + "declares:\n\n " + VOLUME + ) in message + + +def test_the_message_names_both_causes_before_the_exposes(): + message = str(_blocked(_AFTER, [SF_TABLE, VOLUME])) + + causes = message.index("Possible causes:") + upgrade = message.index("applied by a forge-cli release that gave an Iceberg table") + removal = message.index("this change removed a Snowflake-managed Iceberg expose") + recatalog = message.index("or moved one to another catalog") + expose = message.index(" exposes[orders_iceberg]: location.catalog lakekeeper") + assert "does not say which expose it was created for" in message + assert causes < upgrade < removal < recatalog < expose + + +def test_the_remediation_is_kept(): + message = str(_blocked(_AFTER, [SF_TABLE, VOLUME])) + + assert f" tofu -chdir={WORKDIR} state rm {VOLUME}" in message + assert "the resources stay in Snowflake" in message + assert "DROP EXTERNAL VOLUME" in message + + +def test_the_aws_message_still_attributes_the_glue_resources(): + """AWS has per-expose evidence (the Glue table), so its message is as before.""" + location = {"database": "sales", "table": "orders", "bucket": "lake", "path": "orders/"} + location["catalog"] = "lakekeeper" + contract = _contract( + { + "exposeId": "orders", + "binding": {"platform": "aws", "format": "iceberg", "location": location}, + "contract": {"schema": copy.deepcopy(SCHEMA)}, + } + ) + db = "aws_glue_catalog_database.analytics_lake_sales" + table = "aws_glue_catalog_table.analytics_lake_sales_orders" + with pytest.raises(CatalogMoveError) as excinfo: + guard_catalog_moves(AwsIacPlugin(), contract, [db, table]) + + assert str(excinfo.value).startswith( + "iceberg catalog move blocked — this contract's OpenTofu state holds 2 Glue catalog " + "resource(s) for Iceberg table(s) that now live in another catalog:\n\n" + f" exposes[orders]: location.catalog lakekeeper\n\n {db}\n {table}\n\n" + "forge-cli no longer creates a Glue table" + ) diff --git a/tests/policy/test_fw2_confluent_default_catalog.py b/tests/policy/test_fw2_confluent_default_catalog.py new file mode 100644 index 00000000..df28789b --- /dev/null +++ b/tests/policy/test_fw2_confluent_default_catalog.py @@ -0,0 +1,105 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CON-5: a Confluent Tableflow expose with no catalog reads as Glue everywhere. + +The Tableflow IaC publishes such a table to AWS Glue, while the kind table +used to call it ``rest``: policy compile dropped every grant with a false +"cataloged in 'rest'" warning, and dbt-snowflake lost the Glue-linked +database type. Absent and ``catalog: glue`` must now compile the same. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, Optional + +import pytest +import yaml + +from fluid_build.engines.dbt.catalogs_yml import generate_catalogs_yml +from fluid_build.iac import build_module, get_iac_plugin +from fluid_build.iac.providers.confluent import validate_confluent_binding +from fluid_build.policy.compiler import compile_policy + +pytestmark = [pytest.mark.unit] + + +def _contract(catalog: Optional[str]) -> Dict[str, Any]: + loc: Dict[str, Any] = { + "environment_id": "env-123", + "kafka_cluster_id": "lkc-123", + "topic": "orders", + "database": "sales_glue", + "bucket": "acme-tableflow", + "confluent_role_arn": "arn:aws:iam::123456789012:role/tableflow", + "region": "eu-west-1", + } + if catalog is not None: + loc["catalog"] = catalog + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.tableflow", + "name": "Tableflow orders", + "metadata": {"layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": {"platform": "confluent", "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "order_id", "type": "integer"}]}, + } + ], + } + + +def test_policy_compile_grants_the_glue_table_when_catalog_is_absent(): + bindings, warnings = compile_policy(_contract(None)) + assert not any("cataloged in" in w for w in warnings), warnings + assert [(b["provider"], b["resource_type"], b["resource_id"]) for b in bindings] == [ + ("aws", "s3.bucket", "acme-tableflow"), + ("aws", "glue.table", "sales_glue.orders"), + ] + + +def test_policy_compile_is_the_same_for_absent_and_explicit_glue(): + assert compile_policy(_contract(None)) == compile_policy(_contract("glue")) + + +def test_dbt_snowflake_links_the_glue_database_when_catalog_is_absent(): + build = {"engine": "dbt", "execution": {"runtime": {"platform": "snowflake"}}} + absent = generate_catalogs_yml(_contract(None), build) + assert absent == generate_catalogs_yml(_contract("glue"), build) + integration = yaml.safe_load(absent)["catalogs"][0]["write_integrations"][0] + assert integration["adapter_properties"]["catalog_linked_database_type"] == "glue" + + +def test_tableflow_iac_still_publishes_an_absent_catalog_to_glue(): + plugin = get_iac_plugin("confluent") + absent = json.loads(build_module(plugin, _contract(None)))["resource"] + glue = next(iter(absent["confluent_catalog_integration"].values()))["aws_glue"] + assert glue["custom_database"] == "sales_glue" + assert build_module(plugin, _contract(None)) == build_module(plugin, _contract("glue")) + assert validate_confluent_binding(_contract(None)) == validate_confluent_binding( + _contract("glue") + ) + + +def test_a_non_glue_confluent_catalog_is_still_refused(): + errors, _ = validate_confluent_binding(_contract("lakekeeper")) + assert errors and "lakekeeper" in errors[0] diff --git a/tests/policy/test_fw2_tableflow_glue_table.py b/tests/policy/test_fw2_tableflow_glue_table.py new file mode 100644 index 00000000..a3f24f44 --- /dev/null +++ b/tests/policy/test_fw2_tableflow_glue_table.py @@ -0,0 +1,138 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Confluent Tableflow expose is granted the Glue table Tableflow publishes. + +The Tableflow IaC names that table for the Kafka topic (``_topic_name``: +topic > table > exposeId) in ``location.database``, or with no database in +the database Tableflow names after the Kafka cluster id. The compiler used +to read only ``location.table``: a grant on a table nothing publishes, or, +with no table, a database-wide ``glue.table`` grant with ``table: null``. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Sequence + +import pytest + +from fluid_build.iac.providers.confluent import ConfluentIacPlugin +from fluid_build.policy.compiler import SAFE_GLUE_PERMS, compile_policy + +pytestmark = [pytest.mark.unit] + +_LOCATION: Dict[str, Any] = { + "environment_id": "env-123", + "kafka_cluster_id": "lkc-123", + "database": "sales_glue", + "bucket": "acme-tableflow", + "confluent_role_arn": "arn:aws:iam::123456789012:role/tableflow", + "region": "eu-west-1", +} + + +def _contract(permissions: Sequence[str] = ("read",), **location: Any) -> Dict[str, Any]: + loc = {**_LOCATION, **location} + loc = {k: v for k, v in loc.items() if v is not None} + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.tableflow", + "name": "Tableflow orders", + "metadata": {"layer": "Bronze"}, + "accessPolicy": { + "grants": [ + {"principal": "group:analysts@example.com", "permissions": list(permissions)} + ] + }, + "exposes": [ + { + "exposeId": "tableflow_orders", + "kind": "table", + "binding": {"platform": "confluent", "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "order_id", "type": "integer"}]}, + } + ], + } + + +def _glue(bindings: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return [b for b in bindings if b["resource_type"] == "glue.table"] + + +def _published_topic(contract: Dict[str, Any]) -> str: + resources = ConfluentIacPlugin().emit(contract) + return next(iter(resources["confluent_tableflow_topic"].values()))["display_name"] + + +@pytest.mark.parametrize( + "location, table", + [ + ({"topic": "orders.v1", "table": "orders"}, "orders.v1"), + ({"topic": "orders"}, "orders"), + ({"table": "orders"}, "orders"), + ({}, "tableflow_orders"), + ], + ids=["topic-and-table", "topic-only", "table-only", "neither"], +) +def test_the_glue_grant_names_the_table_tableflow_publishes(location, table): + contract = _contract(**location) + bindings, warnings = compile_policy(contract) + assert warnings == [] + assert _published_topic(contract) == table + [glue] = _glue(bindings) + assert (glue["resource_id"], glue["database"], glue["table"]) == ( + f"sales_glue.{table}", + "sales_glue", + table, + ) + assert glue["actions"] == SAFE_GLUE_PERMS["readData"] + assert [b["resource_id"] for b in bindings if b["resource_type"] == "s3.bucket"] == [ + "acme-tableflow" + ] + + +def test_no_database_grants_the_cluster_id_database(): + bindings, warnings = compile_policy(_contract(topic="orders", database=None)) + assert warnings == [] + [glue] = _glue(bindings) + assert (glue["resource_id"], glue["database"], glue["table"]) == ( + "lkc-123.orders", + "lkc-123", + "orders", + ) + + +def test_no_database_and_no_cluster_id_warns_instead_of_granting_a_database(): + bindings, warnings = compile_policy( + _contract(topic="orders", database=None, kafka_cluster_id=None) + ) + assert _glue(bindings) == [] + assert [b["resource_type"] for b in bindings] == ["s3.bucket"] + assert len(warnings) == 1 + assert "'tableflow_orders'" in warnings[0] + assert "no Glue table grant was compiled" in warnings[0] + + +def test_a_write_grant_manages_the_published_table(): + bindings, _ = compile_policy(_contract(permissions=["write"], topic="orders")) + [glue] = _glue(bindings) + assert glue["resource_id"] == "sales_glue.orders" + assert glue["actions"] == SAFE_GLUE_PERMS["manage"] + + +def test_no_tableflow_glue_grant_has_a_null_table(): + for location in ({}, {"topic": "orders"}, {"database": None}): + bindings, _ = compile_policy(_contract(**location)) + assert all(b["table"] for b in _glue(bindings)), bindings diff --git a/tests/providers/test_fw2_catalog_table.py b/tests/providers/test_fw2_catalog_table.py new file mode 100644 index 00000000..edd26adc --- /dev/null +++ b/tests/providers/test_fw2_catalog_table.py @@ -0,0 +1,184 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The catalog-kind table against what each Iceberg catalog needs to start. + +CON-3: Iceberg's DynamoDbCatalog and JdbcCatalog refuse to initialise without +a warehouse (apache-iceberg-1.10.0 DynamoDbCatalog.java:132-134, +JdbcCatalog.java:117-119), so the resolver derives one from an explicit +``location.bucket``. CON-4: BigQueryMetastoreCatalog refuses to initialise +without ``gcp.bigquery.project-id`` (BigQueryMetastoreCatalog.java:84-87). +CON-5: a ``platform: confluent`` expose with no catalog is the Glue table the +Tableflow IaC publishes. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.providers._iceberg_catalog import ( + BIGQUERY_LOCATION, + BIGQUERY_PROJECT_ID, + binding_catalog_kind, + catalog_kind_info, + default_catalog_kind, + is_glue_cataloged, + resolve_iceberg_catalog, +) + +pytestmark = [pytest.mark.unit] + + +def _binding(catalog: Optional[str], platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = {"database": "streaming", "table": "orders", **location} + if catalog is not None: + loc["catalog"] = catalog + return {"platform": platform, "format": "iceberg", "location": loc} + + +# ── the rows: an independent pin of what each sink needs ──────────────────── + + +@pytest.mark.parametrize( + "kind, requires", + [ + ("dynamodb", ("warehouse",)), + ("jdbc", ("uri", "warehouse")), + ("bigquery", ("project",)), + ], +) +def test_sink_requires_matches_the_catalog_initialize_checks(kind, requires): + assert catalog_kind_info(kind).sink_requires == requires + + +# ── CON-3: dynamodb / jdbc derive a warehouse from an explicit bucket ─────── + + +@pytest.mark.parametrize( + "kind, extra", [("dynamodb", {}), ("jdbc", {"uri": "jdbc:postgresql://pg:5432/iceberg"})] +) +@pytest.mark.parametrize( + "platform, expected", + [ + ("aws", "s3://acme-lake/streaming/orders/"), + ("s3", "s3://acme-lake/streaming/orders/"), # an aws alias + ("gcp", "gs://acme-lake/streaming/orders/"), + ], +) +def test_bucket_derives_the_warehouse_with_the_platform_scheme(kind, extra, platform, expected): + r = resolve_iceberg_catalog(_binding(kind, platform, bucket="acme-lake", **extra)) + assert r.warehouse == expected + + +def test_derived_warehouse_honours_location_path(): + r = resolve_iceberg_catalog(_binding("dynamodb", bucket="acme-lake", path="/lake/orders")) + assert r.warehouse == "s3://acme-lake/lake/orders" + + +def test_explicit_warehouse_beats_the_bucket(): + r = resolve_iceberg_catalog(_binding("dynamodb", bucket="acme-lake", warehouse="s3://other/wh")) + assert r.warehouse == "s3://other/wh" + + +@pytest.mark.parametrize( + "binding", + [ + # no bucket: never the account-derived bucket the Glue row falls back to + _binding("dynamodb"), + _binding("jdbc", uri="jdbc:postgresql://pg:5432/iceberg"), + # an unresolved template is not a bucket either + _binding("dynamodb", bucket="{{ env.FW2_NO_SUCH_BUCKET_VAR }}"), + # a platform with no scheme to derive with + _binding("jdbc", "local", bucket="acme-lake", uri="jdbc:sqlite::memory:"), + ], +) +def test_no_explicit_bucket_derives_no_warehouse(binding, monkeypatch): + monkeypatch.delenv("FW2_NO_SUCH_BUCKET_VAR", raising=False) + assert resolve_iceberg_catalog(binding, account_ref="123456789012").warehouse == "" + + +def test_glue_keeps_its_account_fallback(): + r = resolve_iceberg_catalog(_binding("glue"), account_ref="123456789012") + assert r.warehouse == "s3://123456789012-fluid-data/streaming/orders/" + + +@pytest.mark.parametrize("kind", ["hadoop", "hive", "nessie", "rest", "lakekeeper"]) +def test_other_kinds_do_not_derive_from_the_bucket(kind): + assert resolve_iceberg_catalog(_binding(kind, bucket="acme-lake")).warehouse == "" + + +# ── CON-4: bigquery gets its project, location and gs:// warehouse ────────── + + +def _bq(**location: Any) -> Dict[str, Any]: + return _binding("bigquery", "gcp", **location) + + +def test_bigquery_maps_project_and_region_to_its_own_properties(): + r = resolve_iceberg_catalog(_bq(project="acme-proj", region="eu", bucket="acme-lake")) + assert dict(r.extra_catalog_props) == { + BIGQUERY_PROJECT_ID: "acme-proj", + BIGQUERY_LOCATION: "eu", + } + assert BIGQUERY_PROJECT_ID == "gcp.bigquery.project-id" + assert BIGQUERY_LOCATION == "gcp.bigquery.location" + # client.region is an AWS client property + assert r.region is None + + +def test_bigquery_without_region_sets_no_location(): + r = resolve_iceberg_catalog(_bq(project="acme-proj")) + assert dict(r.extra_catalog_props) == {BIGQUERY_PROJECT_ID: "acme-proj"} + + +@pytest.mark.parametrize( + "location, warehouse", + [ + ({"bucket": "acme-lake"}, "gs://acme-lake"), + ({"bucket": "acme-lake", "path": "iceberg"}, "gs://acme-lake/iceberg"), + ({"warehouse": "gs://wh-bucket/root", "bucket": "acme-lake"}, "gs://wh-bucket/root"), + # a foreign scheme is storage the GCP IaC never creates + ({"warehouse": "s3://elsewhere"}, ""), + ({}, ""), + ], +) +def test_bigquery_warehouse_is_the_gs_storage_uri(location, warehouse): + r = resolve_iceberg_catalog(_bq(project="acme-proj", **location)) + assert r.warehouse == warehouse + assert (r.io_impl is not None) == bool(warehouse) + + +# ── CON-5: confluent's default catalog is Glue, as Tableflow publishes ────── + + +def test_confluent_default_catalog_is_glue(): + binding = {"platform": "confluent", "format": "iceberg", "location": {"database": "d"}} + assert default_catalog_kind(binding) == "glue" + assert binding_catalog_kind(binding) == "glue" + assert is_glue_cataloged(binding) + + +@pytest.mark.parametrize( + "platform, expected", + [("aws", "glue"), ("snowflake", "snowflake-managed"), ("gcp", "rest"), ("local", "rest")], +) +def test_other_platform_defaults_are_unchanged(platform, expected): + assert default_catalog_kind({"platform": platform}) == expected + + +def test_confluent_explicit_catalog_still_wins(): + binding = {"platform": "confluent", "location": {"catalog": "lakekeeper"}} + assert binding_catalog_kind(binding) == "lakekeeper" diff --git a/tests/providers/test_fw2_sink_expose.py b/tests/providers/test_fw2_sink_expose.py new file mode 100644 index 00000000..495b875e --- /dev/null +++ b/tests/providers/test_fw2_sink_expose.py @@ -0,0 +1,216 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LOGIC-1: the build->expose join for a sink config forge-cli derives. + +``resolve_iceberg_sink_exposes`` picks the Iceberg sink exposes from the +build's outputs. Kafka Connect derives one ``iceberg.tables`` entry, so it +resolves exactly one expose; embedded Debezium Server derives one +``table-namespace`` and one catalog, so it resolves several when they share a +database and resolve to one catalog. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.providers._iceberg_catalog import ( + find_iceberg_expose_binding, + resolve_iceberg_sink_exposes, +) + +pytestmark = [pytest.mark.unit] + + +def _expose(eid: str, database: str, table: str, **location: Any) -> Dict[str, Any]: + return { + "exposeId": eid, + "binding": { + "platform": "aws", + "format": "iceberg", + "location": {"database": database, "table": table, "bucket": "lake", **location}, + }, + } + + +def _ids(exposes) -> List[str]: + return [e["exposeId"] for e in exposes] + + +def _build(outputs: Optional[Any]) -> Dict[str, Any]: + build: Dict[str, Any] = {"id": "ingest"} + if outputs is not None: + build["outputs"] = outputs + return build + + +ORDERS = _expose("orders", "sales", "orders") +REFUNDS = _expose("refunds", "sales", "refunds") +CONFLUENT = { + "exposeId": "managed", + "binding": {"platform": "confluent", "format": "iceberg", "location": {"table": "m"}}, +} +PARQUET = {"exposeId": "raw", "binding": {"platform": "aws", "format": "parquet"}} + + +# ── Kafka Connect (one table) ─────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "exposes, outputs, expected", + [ + ([ORDERS, REFUNDS], ["refunds"], ["refunds"]), + ([ORDERS, REFUNDS], ["orders"], ["orders"]), + ([ORDERS, REFUNDS, PARQUET], ["raw", "refunds"], ["refunds"]), + ([ORDERS], None, ["orders"]), + ([ORDERS], [], ["orders"]), + ([ORDERS], "orders", ["orders"]), + ([CONFLUENT, ORDERS], None, ["orders"]), + ], + ids=[ + "second-expose", + "first-expose", + "skips-non-iceberg-output", + "single-no-outputs", + "single-empty-outputs", + "outputs-as-a-string", + "confluent-is-not-a-candidate", + ], +) +def test_one_table_resolves(exposes, outputs, expected): + target, error = resolve_iceberg_sink_exposes({"exposes": exposes}, _build(outputs)) + assert error is None + assert _ids(target) == expected + + +@pytest.mark.parametrize( + "exposes, outputs, message", + [ + ([ORDERS, PARQUET], ["raw"], "outputs ['raw'] name none of the Iceberg sink exposes"), + ([ORDERS, CONFLUENT], ["managed"], "outputs ['managed'] name none"), + ([ORDERS, REFUNDS], ["orders", "refunds"], "outputs ['orders', 'refunds'] name 2 of"), + ([ORDERS, REFUNDS], None, "declares no outputs, and the contract has 2"), + ], + ids=["non-iceberg", "confluent-managed", "two-outputs", "no-outputs"], +) +def test_one_table_refuses(exposes, outputs, message): + target, error = resolve_iceberg_sink_exposes({"exposes": exposes}, _build(outputs)) + assert target == () + assert error is not None and message in error + assert error.startswith("iceberg sink (build 'ingest'): ") + assert "['orders (sales.orders)'" in error + assert "a derived Kafka Connect sink writes one expose" in error + + +def test_no_iceberg_sink_expose_is_left_to_the_validator(): + assert resolve_iceberg_sink_exposes({"exposes": [PARQUET]}, _build(["raw"])) == ((), None) + assert resolve_iceberg_sink_exposes({}, _build(None), namespace=True) == ((), None) + + +# ── embedded Debezium Server (one namespace, one catalog) ─────────────────── + + +@pytest.mark.parametrize( + "exposes, outputs, expected", + [ + ([ORDERS, REFUNDS], ["orders", "refunds"], ["orders", "refunds"]), + ([ORDERS, REFUNDS], None, ["orders", "refunds"]), + ([ORDERS, REFUNDS], ["refunds"], ["refunds"]), + ([ORDERS, _expose("refunds", "finance", "refunds")], ["refunds"], ["refunds"]), + ], + ids=["both-outputs", "no-outputs", "one-output", "one-output-other-database"], +) +def test_namespace_resolves(exposes, outputs, expected): + target, error = resolve_iceberg_sink_exposes( + {"exposes": exposes}, _build(outputs), namespace=True + ) + assert error is None + assert _ids(target) == expected + + +def test_namespace_glue_warehouse_is_per_table_not_a_catalog_difference(): + # get_iceberg_warehouse derives s3://lake/sales/
/ for each one + target, error = resolve_iceberg_sink_exposes( + {"exposes": [ORDERS, REFUNDS]}, _build(None), namespace=True + ) + assert error is None and len(target) == 2 + + +_LK = {"uri": "http://lakekeeper:8181/catalog", "warehouse": "wh"} + + +@pytest.mark.parametrize( + "refunds, message", + [ + (_expose("refunds", "finance", "refunds"), "sit in the databases ['finance', 'sales']"), + ( + _expose("refunds", "sales", "refunds", catalog="lakekeeper", **_LK), + "(their catalog, warehouse, uri differ)", + ), + (_expose("refunds", "sales", "refunds", region="eu-west-1"), "(their region differ)"), + ], + ids=["two-databases", "two-catalog-kinds", "two-regions"], +) +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["outputs", "no-outputs"]) +def test_namespace_refuses(refunds, message, outputs): + target, error = resolve_iceberg_sink_exposes( + {"exposes": [ORDERS, refunds]}, _build(outputs), namespace=True + ) + assert target == () + assert error is not None and message in error + assert "a derived Debezium Server sink" in error + assert "hand-write the sink config (properties.debezium.server.sink.config)" in error + + +@pytest.mark.parametrize( + "location, message", + [ + ({"warehouse": "other"}, "(their warehouse differ)"), + ({"uri": "http://other:8181/catalog"}, "(their uri differ)"), + ({"catalog": "polaris"}, "(their catalog differ)"), + ], + ids=["warehouse", "uri", "kind"], +) +def test_namespace_rest_catalogs_must_match(location, message): + orders = _expose("orders", "sales", "orders", catalog="lakekeeper", **_LK) + refunds = _expose("refunds", "sales", "refunds", **{"catalog": "lakekeeper", **_LK, **location}) + contract = {"exposes": [orders, refunds]} + target, error = resolve_iceberg_sink_exposes(contract, _build(None), namespace=True) + assert target == () and error is not None and message in error + same = { + "exposes": [orders, _expose("refunds", "sales", "refunds", catalog="lakekeeper", **_LK)] + } + assert _ids(resolve_iceberg_sink_exposes(same, _build(None), namespace=True)[0]) == [ + "orders", + "refunds", + ] + + +def test_namespace_outputs_naming_none_are_refused(): + target, error = resolve_iceberg_sink_exposes( + {"exposes": [ORDERS, PARQUET]}, _build(["raw"]), namespace=True + ) + assert target == () + assert error is not None and "outputs ['raw'] name none of the Iceberg sink exposes" in error + + +# ── the first-expose helper a config that names its tables is checked against ─ + + +def test_find_iceberg_expose_binding_is_still_the_first_iceberg_sink_expose(): + assert find_iceberg_expose_binding({"exposes": [CONFLUENT, ORDERS, REFUNDS]}) == ( + ORDERS["binding"] + ) diff --git a/tests/test_mcp_sdk_rename_guard.py b/tests/test_mcp_sdk_rename_guard.py index ac8f8bc7..9cb16ada 100644 --- a/tests/test_mcp_sdk_rename_guard.py +++ b/tests/test_mcp_sdk_rename_guard.py @@ -26,7 +26,7 @@ does not find the fields. Neither raises, so neither shows up as a test failure elsewhere. That is what this file is for. -The audit behind the ``mcp>=1.20,<3.0`` ceiling found zero of either. These +The audit behind the ``<3.0`` ceiling of ``mcp>=1.22,<3.0`` found zero of either. These tests pin that result so it cannot drift back unnoticed, in the same spirit as ``tests/perf/test_startup_budget.py``, which guards the cold-path import budget an ordinary test cannot express.