From 9af74508e9ac23f8024edce225aa9d19b830f9ac Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:14:00 +0200 Subject: [PATCH 01/19] fix(policy,iac): show policy compile warnings, fail on a compiler crash, and word the Snowflake volume guard without blaming an expose RT-707-3: `fluid policy compile` wrote its warnings only into bindings.json and logged a count at INFO (routed to DEBUG), so a grant that compiled to no table grant printed nothing and exited 0. Each warning now goes to stderr through the CLI warning helper, exit stays 0, and `fluid policy apply` echoes the warnings stored in bindings.json. "No grants found in accessPolicy" (a legitimate no-op) stays in the file and is not shown as a WARNING on compile, apply or stage 3. A crash inside compile_policy was swallowed into {"bindings": {"bindings": []}} and exit 0. It now fails with the new policy_compiler_crashed error, writes nothing, and names the schema type errors at the values the compiler reads (the grants, their permissions, each binding and its location), or the compiler's own exception when there are none. Schema errors elsewhere, such as an extra key on a grant, are not named. The error routes to the policy compile page's Errors section. JRN-707-6: the Snowflake EXTERNAL VOLUME guard detects what it did before. Its message no longer says forge-cli stopped creating the volume or lists the moved exposes as the tables it was held for: the volume is keyed by the contract, so the message names both possible causes (an upgrade from a release that gave a non-Snowflake-managed catalog a volume, or a Snowflake-managed expose removed or re-cataloged in this change) and keeps the tofu state rm remediation. The AWS message, which has per-expose evidence, is unchanged. --- fluid_build/_error_catalog.py | 13 ++ fluid_build/_errors.py | 2 + fluid_build/cli/policy_apply.py | 13 +- fluid_build/cli/policy_compile.py | 68 +++++- fluid_build/iac/catalog_moves.py | 47 ++-- fluid_build/policy/compiler.py | 8 +- .../test_fw2_policy_compile_read_blocks.py | 214 ++++++++++++++++++ .../test_fw2_policy_compile_schema_error.py | 103 +++++++++ tests/cli/test_fw2_policy_compile_warnings.py | 204 +++++++++++++++++ .../iac/test_fw2_snowflake_volume_message.py | 151 ++++++++++++ 10 files changed, 800 insertions(+), 23 deletions(-) create mode 100644 tests/cli/test_fw2_policy_compile_read_blocks.py create mode 100644 tests/cli/test_fw2_policy_compile_schema_error.py create mode 100644 tests/cli/test_fw2_policy_compile_warnings.py create mode 100644 tests/iac/test_fw2_snowflake_volume_message.py diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index f807b5d7..e013006d 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -341,6 +341,19 @@ def slug_for(event: str) -> str: ["Check the agent-policy block in the contract; run 'fluid policy check '"], "sovereignty", ), + "policy_compiler_crashed": ( + [ + ( + "Run 'fluid validate ': policy compile reads accessPolicy and " + + "exposes without validating them against the contract schema" + ), + ( + "If the contract validates, re-run with 'fluid --log-level DEBUG policy compile " + + "' to see the compiler's traceback" + ), + ], + "policy-compile#errors", + ), "policy_apply_failed": ( ["Run 'fluid policy check ' to surface the offending rule before apply"], "sovereignty", diff --git a/fluid_build/_errors.py b/fluid_build/_errors.py index dce8617c..af65a38d 100644 --- a/fluid_build/_errors.py +++ b/fluid_build/_errors.py @@ -83,6 +83,8 @@ "infra#drift": "advanced/typed-cli-errors.html#governance", # the safety gate's own section, which shows its message and remediation "iceberg-catalog-move": "cli/apply.html#iceberg-catalog-move-guard", + # the command's own Errors section + "policy-compile#errors": "cli/policy-compile.html#errors", } _DOC_FALLBACK = "advanced/production-troubleshooting.html" diff --git a/fluid_build/cli/policy_apply.py b/fluid_build/cli/policy_apply.py index 2b4587b4..330f5444 100644 --- a/fluid_build/cli/policy_apply.py +++ b/fluid_build/cli/policy_apply.py @@ -22,7 +22,7 @@ from ..observability.tracing import traced_stage as _traced_stage from ._common import CLIError, build_provider -from ._logging import info +from ._logging import info, warn from .console import cprint COMMAND = "policy-apply" @@ -87,6 +87,17 @@ def run(args, logger: logging.Logger) -> int: with open(bindings_path, encoding="utf-8") as f: data = json.load(f) + # The compiler's warnings, echoed here too: a grant that compiled to + # no binding (see ``fluid policy compile``) is not enforced by this + # step either, and the run that applies is often not the one that + # compiled. A contract with no grants leaves nothing unenforced. + from fluid_build.policy.compiler import NO_GRANTS + + stored = data.get("warnings") if isinstance(data, dict) else None + for warning in stored if isinstance(stored, list) else (): + if warning != NO_GRANTS: + warn(logger, "policy_bindings_warning", warning=warning, bindings=args.bindings) + # An empty bindings file is a legitimate no-op, not an error: a # contract with no ``accessPolicy`` grants — every raw bronze / SDP # acquisition product — compiles to zero bindings. There is nothing diff --git a/fluid_build/cli/policy_compile.py b/fluid_build/cli/policy_compile.py index 13c805db..77ecc02b 100644 --- a/fluid_build/cli/policy_compile.py +++ b/fluid_build/cli/policy_compile.py @@ -17,9 +17,11 @@ import argparse import logging import os +import re +from typing import Any, List from ._common import CLIError, load_contract_with_overlay, write_json -from ._logging import info +from ._logging import info, warn COMMAND = "policy-compile" @@ -47,23 +49,79 @@ def register(subparsers: argparse._SubParsersAction): _add_arguments(p) +#: The values ``compile_policy`` reads as a mapping or a list. A wrong type at +#: one of these paths is a value the compiler cannot read. Any other schema +#: error (an extra key on a grant, a wrong type under ``metadata``) is not +#: named: it did not make the compiler fail. +_COMPILER_READ_PATHS = re.compile( + r"accessPolicy(?:\.grants(?:\[\d+\](?:\.permissions)?)?)?" + r"|exposes\[\d+\]\.binding(?:\.location)?" +) + + +def _compiler_schema_errors(contract: Any) -> List[str]: + """The schema type errors ``fluid validate`` reports where the compiler reads. + + ``policy compile`` does not schema-validate, so a grant or binding of the + wrong type reaches the compiler and fails there as a Python error that + names neither the value nor the fix. Read only after the compiler failed; + ``[]`` when the schema finds no such error or cannot be run. + """ + try: + from fluid_build.schema_manager import FluidSchemaManager + + result = FluidSchemaManager().validate_contract(contract, offline_only=True) + except Exception: # the compiler's own error is reported instead + return [] + # An error starts with its path: ``accessPolicy.grants[0]: ...``, + # ``exposes[0].binding: ...``. + found = [] + for error in result.errors: + path, _, message = error.partition(": ") + if _COMPILER_READ_PATHS.fullmatch(path) and " is not of type " in message: + found.append(error) + return found + + +def _compile_failed(contract: Any, exc: Exception) -> CLIError: + """The ``policy_compiler_crashed`` error for a crash inside ``compile_policy``.""" + schema_errors = _compiler_schema_errors(contract) + if schema_errors: + error = "policy compile failed on contract values of the wrong type: " + "; ".join( + schema_errors + ) + else: + error = f"policy compiler failed: {type(exc).__name__}: {exc}" + return CLIError(1, "policy_compiler_crashed", {"error": error}) + + def run(args, logger: logging.Logger) -> int: import traceback try: c = load_contract_with_overlay(args.contract, getattr(args, "env", None), logger) try: - from fluid_build.policy.compiler import compile_policy + from fluid_build.policy.compiler import NO_GRANTS, compile_policy bindings, warnings = compile_policy(c) except Exception as e: - logger.error(f"Policy compiler error: {e}") - logger.error(traceback.format_exc()) - bindings, warnings = {"bindings": []}, [f"policy compiler failed: {str(e)}"] + # Fail closed. This used to write ``{"bindings": []}`` and exit 0, + # so a crash read as "no grants to enforce" to the next stage. + # Nothing is written: a bindings file from an earlier run stays as + # it was, and the non-zero exit stops the pipeline before apply. + logger.debug(traceback.format_exc()) + raise _compile_failed(c, e) from e out_dir = os.path.dirname(args.out) if out_dir: # Only create dir if path has a directory component os.makedirs(out_dir, exist_ok=True) write_json(args.out, {"bindings": bindings, "warnings": warnings}) + # A warning can be the only sign that a grant compiled to nothing (an + # Iceberg table in a catalog the compiler cannot grant on), so it goes + # to the console, not only into the file. Warnings keep exit 0. A + # contract with no grants leaves nothing unenforced: not a WARNING. + for warning in warnings: + if warning != NO_GRANTS: + warn(logger, "policy_compile_warning", warning=warning, out=args.out) info(logger, "policy_compiled", out=args.out, warnings=len(warnings)) return 0 except CLIError: diff --git a/fluid_build/iac/catalog_moves.py b/fluid_build/iac/catalog_moves.py index 4514e511..9cfffcd7 100644 --- a/fluid_build/iac/catalog_moves.py +++ b/fluid_build/iac/catalog_moves.py @@ -128,7 +128,9 @@ class CatalogMoveSpec: ``addresses_before`` returns addresses the old release created for a moved binding that do not depend on its location, for a resource the before emit cannot re-derive once the location changed too; the default is none. - The prose fields fill the error message. + The prose fields fill the error message. With no ``evidence_types``, the + message does not attribute the resources to the moved exposes: + ``consequence`` says why they may be in state, and the exposes follow it. """ cloud: str @@ -138,7 +140,8 @@ class CatalogMoveSpec: as_before: Callable[[MutableMapping[str, Any]], None] #: Counted noun for the resources, e.g. "Glue catalog resource(s)". noun: str - #: Why the plan would destroy them, and what that destroy does. + #: Why the plan would destroy them, and what that destroy does. With no + #: ``evidence_types`` it also introduces the expose list printed after it. consequence: str #: Where ``state rm`` leaves them, e.g. "AWS". stays_in: str @@ -276,17 +279,20 @@ def _snowflake_volume_before( as_before=_back_into_snowflake, noun="Snowflake EXTERNAL VOLUME(s)", consequence=( - "forge-cli no longer creates an EXTERNAL VOLUME for an Iceberg table in a catalog " - "Snowflake does not manage (dbt now writes it as an externally cataloged table, " - "not a Snowflake-managed one on a volume), so applying now would plan to DROP " - "these volumes, and any Snowflake-managed Iceberg table an earlier dbt run wrote " - "onto one still uses it." + "The volume is named for the contract, not for an expose, so the state does not " + "say which expose it was created for. Possible causes: this contract was applied " + "by a forge-cli release that gave an Iceberg table in a catalog Snowflake does not " + "manage an EXTERNAL VOLUME (this release gives it none, and dbt writes it as an " + "externally cataloged table); or this change removed a Snowflake-managed Iceberg " + "expose, or moved one to another catalog. Applying now would plan to DROP the " + "volume, and any Snowflake-managed Iceberg table written onto it still uses it.\n\n" + "Iceberg exposes whose catalog earlier releases gave an EXTERNAL VOLUME:" ), stays_in="Snowflake", afterwards=( "Drop a volume by hand (DROP EXTERNAL VOLUME) only once no Iceberg table uses it. " - "If the table belongs in Snowflake's own catalog, remove location.catalog (or set " - "it to 'snowflake') instead." + "If an Iceberg table belongs in Snowflake's own catalog, remove its " + "location.catalog (or set it to 'snowflake') instead." ), addresses_before=_snowflake_volume_before, ) @@ -485,15 +491,24 @@ def guard_catalog_moves( if not addresses: return commands = _state_rm_commands(addresses, workdir=workdir) + expose_lines = "\n".join(f" exposes[{e}]: location.catalog {k}" for e, k in exposes) + address_lines = "\n".join(f" {a}" for a in addresses) + if spec.evidence_types: + found = ( + f"{len(addresses)} {spec.noun} for Iceberg table(s) that now live in another " + f"catalog:\n\n{expose_lines}\n\n{address_lines}\n\n{spec.consequence}" + ) + else: + # No resource in state ties these to an expose, so the moved exposes + # are one possible cause, listed after the prose that says so. + found = ( + f"{len(addresses)} {spec.noun} that this contract's configuration no longer " + f"declares:\n\n{address_lines}\n\n{spec.consequence}\n\n{expose_lines}" + ) raise CatalogMoveError( "iceberg catalog move blocked — this contract's OpenTofu state holds " - f"{len(addresses)} {spec.noun} for Iceberg table(s) that now live in another " - "catalog:\n\n" - + "\n".join(f" exposes[{e}]: location.catalog {k}" for e, k in exposes) - + "\n\n" - + "\n".join(f" {a}" for a in addresses) - + f"\n\n{spec.consequence}\n\n" - "Drop each from this contract's state, then re-run apply:\n\n" + + found + + "\n\nDrop each from this contract's state, then re-run apply:\n\n" + "\n".join(f" {command}" for command in commands) + "\n\n`tofu state rm` touches ZERO bytes of infrastructure: the resources stay in " f"{spec.stays_in}, and only this contract's claim on them is released. " + spec.afterwards, diff --git a/fluid_build/policy/compiler.py b/fluid_build/policy/compiler.py index 1b6d64a0..6622ec40 100644 --- a/fluid_build/policy/compiler.py +++ b/fluid_build/policy/compiler.py @@ -51,6 +51,12 @@ "manage": ["INSERT", "UPDATE", "DELETE", "SELECT"], } +#: The message for a contract with no grants. It stays in ``warnings`` (and +#: so in ``bindings.json``), but it reports a legitimate no-op, not a grant +#: left unenforced, so ``fluid policy compile`` / ``apply`` do not show it as +#: a WARNING. +NO_GRANTS = "No grants found in accessPolicy" + def compile_policy(contract: dict) -> Tuple[List[Dict[str, Any]], List[str]]: """Compile accessPolicy from contract into provider IAM bindings. @@ -70,7 +76,7 @@ def compile_policy(contract: dict) -> Tuple[List[Dict[str, Any]], List[str]]: grants = (contract.get("accessPolicy") or {}).get("grants", []) if not grants: - warnings.append("No grants found in accessPolicy") + warnings.append(NO_GRANTS) return bindings, warnings for g in grants: diff --git a/tests/cli/test_fw2_policy_compile_read_blocks.py b/tests/cli/test_fw2_policy_compile_read_blocks.py new file mode 100644 index 00000000..09f168da --- /dev/null +++ b/tests/cli/test_fw2_policy_compile_read_blocks.py @@ -0,0 +1,214 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A ``policy compile`` crash names the value that crashed it, and a contract +with no grants is not a warning. + +A crash inside ``compile_policy`` is reported with the schema type errors +``fluid validate`` finds at the values the compiler reads (the grants, their +permissions, each expose's binding and location). A schema error elsewhere, +such as an extra key on a grant, did not make the compiler fail and is not +named. Once compile warnings reached the console, a contract with no grants (a +legitimate no-op) must not print one on compile, on apply, or in stage 3. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import logging +from pathlib import Path +from typing import Any, Dict, List + +import pytest +import yaml + +from fluid_build.cli import policy_apply, policy_compile +from fluid_build.cli._common import CLIError + +pytestmark = pytest.mark.unit + +_LOGGER = "test.fw2.policy.blocks" + +_CONTRACT: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "analytics.pol", + "name": "Pol", + "metadata": {"owner": {"team": "data"}, "layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": {"bucket": "acme-lake", "path": "orders/", "region": "eu-west-1"}, + }, + "contract": {"schema": [{"name": "order_id", "type": "integer", "required": True}]}, + } + ], +} + +_BINDING_ERROR = "exposes[0].binding: 'nope' is not of type 'object'" + + +def _write(tmp_path: Path, doc: Dict[str, Any]) -> Path: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + return path + + +def _compile_error(tmp_path: Path, doc: Dict[str, Any]) -> str: + args = argparse.Namespace(contract=str(_write(tmp_path, doc)), out=str(tmp_path / "b.json")) + with pytest.raises(CLIError) as excinfo: + policy_compile.run(args, logging.getLogger(_LOGGER)) + assert excinfo.value.event == "policy_compiler_crashed" + return str(excinfo.value.context["error"]) + + +def _warning_events(caplog) -> List[Dict[str, Any]]: + out = [] + for record in caplog.records: + if record.levelno < logging.WARNING: + continue + try: + out.append(json.loads(record.getMessage())) + except ValueError: + out.append({"message": record.getMessage()}) + return out + + +def test_a_string_binding_is_reported_as_its_schema_error(tmp_path): + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"] = "nope" + + error = _compile_error(tmp_path, doc) + + assert _BINDING_ERROR in error + assert "AttributeError" not in error + + +def test_an_unrelated_grant_error_is_not_named(tmp_path): + """The extra ``note`` key does not crash the compiler; the binding does. + Only the binding is named.""" + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"] = "nope" + doc["accessPolicy"]["grants"][0]["note"] = "x" + + error = _compile_error(tmp_path, doc) + + assert error == f"policy compile failed on contract values of the wrong type: {_BINDING_ERROR}" + assert "note" not in error + + +@pytest.mark.parametrize( + "mutate, expected", + [ + ( + lambda d: d.update(accessPolicy=["x"]), + "accessPolicy: ['x'] is not of type 'object'", + ), + ( + lambda d: d["accessPolicy"].update(grants="g"), + "accessPolicy.grants: 'g' is not of type 'array'", + ), + ( + lambda d: d["accessPolicy"]["grants"][0].update(permissions=None), + "accessPolicy.grants[0].permissions: None is not of type 'array'", + ), + ( + lambda d: d["exposes"][0]["binding"].update(location=None), + "exposes[0].binding.location: None is not of type 'object'", + ), + ], + ids=["accessPolicy", "grants", "permissions", "location"], +) +def test_each_value_the_compiler_reads_is_named_when_it_crashes(tmp_path, mutate, expected): + """Each of these crashes ``compile_policy`` today; the error names the value.""" + doc = copy.deepcopy(_CONTRACT) + mutate(doc) + + assert _compile_error(tmp_path, doc) == ( + f"policy compile failed on contract values of the wrong type: {expected}" + ) + + +def test_a_schema_error_outside_the_blocks_the_compiler_reads_is_not_named(tmp_path, monkeypatch): + def _boom(contract): + raise KeyError("location") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + doc = copy.deepcopy(_CONTRACT) + doc["metadata"]["layer"] = 7 + + assert _compile_error(tmp_path, doc) == "policy compiler failed: KeyError: 'location'" + + +def test_stage_3_does_not_warn_for_a_contract_with_no_grants(tmp_path, caplog): + """``fluid generate artifacts`` compiles the policy through the same run().""" + from fluid_build.forge.core.artifact_fanout import _emit_policies + + caplog.set_level(logging.DEBUG) + doc = copy.deepcopy(_CONTRACT) + del doc["accessPolicy"] + + _emit_policies(_write(tmp_path, doc), tmp_path / "policy", logging.getLogger(_LOGGER)) + + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + + +def test_a_contract_with_no_grants_compiles_and_applies_without_a_warning( + tmp_path, monkeypatch, caplog +): + from fluid_build.policy.compiler import NO_GRANTS + + caplog.set_level(logging.DEBUG, logger=_LOGGER) + doc = copy.deepcopy(_CONTRACT) + del doc["accessPolicy"] + out = tmp_path / "policy" / "bindings.json" + args = argparse.Namespace(contract=str(_write(tmp_path, doc)), out=str(out)) + + assert policy_compile.run(args, logging.getLogger(_LOGGER)) == 0 + # The file still says why it is empty. + assert json.loads(out.read_text())["warnings"] == [NO_GRANTS] + + monkeypatch.chdir(tmp_path) + apply_args = argparse.Namespace( + bindings="policy/bindings.json", mode="check", provider=None, project=None + ) + assert policy_apply.run(apply_args, logging.getLogger(_LOGGER)) == 0 + + assert _warning_events(caplog) == [] + + +def test_grants_that_compile_to_nothing_still_warn(tmp_path, caplog): + """The control: a grant over an expose the compiler cannot grant on.""" + caplog.set_level(logging.DEBUG, logger=_LOGGER) + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"] = { + "platform": "azure", + "format": "delta", + "location": {"container": "lake"}, + } + args = argparse.Namespace(contract=str(_write(tmp_path, doc)), out=str(tmp_path / "b.json")) + + assert policy_compile.run(args, logging.getLogger(_LOGGER)) == 0 + + shown = [e.get("warning") for e in _warning_events(caplog)] + assert "No IAM bindings generated from contract" in shown diff --git a/tests/cli/test_fw2_policy_compile_schema_error.py b/tests/cli/test_fw2_policy_compile_schema_error.py new file mode 100644 index 00000000..2c8ebee3 --- /dev/null +++ b/tests/cli/test_fw2_policy_compile_schema_error.py @@ -0,0 +1,103 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A ``policy compile`` crash names the block at fault and the command to run. + +``policy compile`` does not schema-validate the contract, so a grant written as +a bare string reaches ``compile_policy`` and fails there with +``AttributeError: 'str' object has no attribute 'get'``. Once that crash +became the command's failure (RT-707-3), it must name the value at fault and +point at ``fluid validate``, not at the agent-policy block or the sovereignty +page that ``policy_compile_failed`` routes to. +""" + +from __future__ import annotations + +import argparse +import copy +import logging +from pathlib import Path +from typing import Any, Dict + +import pytest +import yaml + +from fluid_build._errors import _DOC_BASE +from fluid_build.cli import policy_compile +from fluid_build.cli._common import CLIError + +pytestmark = pytest.mark.unit + +_CONTRACT: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "analytics.pol", + "name": "Pol", + "metadata": {"owner": {"team": "data"}, "layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": {"bucket": "acme-lake", "path": "orders/", "region": "eu-west-1"}, + }, + "contract": {"schema": [{"name": "order_id", "type": "integer", "required": True}]}, + } + ], +} + + +def _run(tmp_path: Path, doc: Dict[str, Any]) -> CLIError: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + args = argparse.Namespace(contract=str(path), out=str(tmp_path / "o" / "b.json")) + with pytest.raises(CLIError) as excinfo: + policy_compile.run(args, logging.getLogger("test.fw2.policy")) + return excinfo.value + + +def test_a_malformed_grant_is_reported_as_the_schema_error(tmp_path): + doc = copy.deepcopy(_CONTRACT) + doc["accessPolicy"]["grants"] = ["group:analysts@example.com"] + + err = _run(tmp_path, doc) + + assert err.event == "policy_compiler_crashed" + assert err.context["error"] == ( + "policy compile failed on contract values of the wrong type: " + "accessPolicy.grants[0]: 'group:analysts@example.com' is not of type 'object'" + ) + assert "AttributeError" not in err.context["error"] + assert not (tmp_path / "o" / "b.json").exists() + + +def test_the_suggestion_points_at_fluid_validate_not_the_agent_policy_block(tmp_path, monkeypatch): + """A crash the schema does not explain keeps the compiler's own error.""" + + def _boom(contract): + raise KeyError("location") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + err = _run(tmp_path, _CONTRACT) + + assert err.context["error"] == "policy compiler failed: KeyError: 'location'" + assert "fluid validate " in err.suggestions[0] + assert "accessPolicy" in err.suggestions[0] + assert "agent-policy" not in " ".join(err.suggestions) + assert err.docs_url == f"{_DOC_BASE}/cli/policy-compile.html#errors" diff --git a/tests/cli/test_fw2_policy_compile_warnings.py b/tests/cli/test_fw2_policy_compile_warnings.py new file mode 100644 index 00000000..9494f869 --- /dev/null +++ b/tests/cli/test_fw2_policy_compile_warnings.py @@ -0,0 +1,204 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid policy compile`` shows its warnings, and fails on a compiler crash. + +RT-707-3: a grant on an Iceberg table in a catalog the compiler cannot grant +on (``catalog: lakekeeper`` on AWS) compiles to no table grant and a warning. +The warning went only into ``bindings.json``; the console got an INFO count +that is routed to DEBUG, so the command printed nothing and exited 0, and +``fluid policy apply`` never read the warnings either. A crash inside +``compile_policy`` was swallowed into a bindings file and exit 0 as well. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, List + +import pytest +import yaml + +from fluid_build.cli import policy_apply, policy_compile +from fluid_build.cli._common import CLIError + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[2] + +_LAKEKEEPER_GRANT: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "analytics.pol", + "name": "Pol", + "metadata": {"owner": {"team": "data"}, "layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "catalog": "lakekeeper", + "uri": "http://lakekeeper:8181/catalog", + "warehouse": "analytics", + "database": "streaming", + "table": "orders", + "bucket": "acme-lake", + "region": "eu-west-1", + }, + }, + "contract": {"schema": [{"name": "order_id", "type": "integer", "required": True}]}, + } + ], +} + +_NO_TABLE_GRANT = "so no table grant was compiled for group:analysts@example.com" + + +def _write_contract(tmp_path: Path, doc: Dict[str, Any]) -> Path: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + return path + + +def _warning_events(caplog, name: str) -> List[Dict[str, Any]]: + out = [] + for record in caplog.records: + if record.levelno != logging.WARNING: + continue + try: + doc = json.loads(record.getMessage()) + except ValueError: + continue + if doc.get("message") == name: + out.append(doc) + return out + + +def test_compile_shows_each_warning_at_warning_level_and_exits_0(tmp_path, caplog): + caplog.set_level(logging.DEBUG, logger="test.fw2.policy") + out = tmp_path / "policy" / "bindings.json" + args = argparse.Namespace( + contract=str(_write_contract(tmp_path, _LAKEKEEPER_GRANT)), out=str(out) + ) + + assert policy_compile.run(args, logging.getLogger("test.fw2.policy")) == 0 + + stored = json.loads(out.read_text())["warnings"] + shown = _warning_events(caplog, "policy_compile_warning") + assert [e["warning"] for e in shown] == stored + assert any(_NO_TABLE_GRANT in w for w in stored) + + +def test_a_compiler_crash_fails_and_writes_no_bindings(tmp_path, monkeypatch): + def _boom(contract): + raise KeyError("location") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + out = tmp_path / "bindings.json" + args = argparse.Namespace( + contract=str(_write_contract(tmp_path, _LAKEKEEPER_GRANT)), out=str(out) + ) + + with pytest.raises(CLIError) as excinfo: + policy_compile.run(args, logging.getLogger("test.fw2.policy")) + + assert excinfo.value.exit_code == 1 + assert excinfo.value.event == "policy_compiler_crashed" + assert "policy compiler failed: KeyError" in excinfo.value.context["error"] + assert not out.exists() + + +def test_a_compiler_crash_leaves_an_earlier_bindings_file_as_it_was(tmp_path, monkeypatch): + out = tmp_path / "bindings.json" + out.write_text('{"bindings": [], "warnings": ["earlier"]}', encoding="utf-8") + + def _boom(contract): + raise RuntimeError("compiler bug") + + monkeypatch.setattr("fluid_build.policy.compiler.compile_policy", _boom) + args = argparse.Namespace( + contract=str(_write_contract(tmp_path, _LAKEKEEPER_GRANT)), out=str(out) + ) + + with pytest.raises(CLIError): + policy_compile.run(args, logging.getLogger("test.fw2.policy")) + assert out.read_text(encoding="utf-8") == '{"bindings": [], "warnings": ["earlier"]}' + + +def test_apply_echoes_the_stored_warnings(tmp_path, monkeypatch, caplog): + """An empty bindings list is the no-op path, which resolves no provider; + the warnings are echoed before it.""" + monkeypatch.chdir(tmp_path) + caplog.set_level(logging.DEBUG, logger="test.fw2.policy") + warning = "Iceberg expose 'orders' is cataloged in 'lakekeeper', not AWS Glue, " + ( + _NO_TABLE_GRANT + ) + path = tmp_path / "bindings.json" + path.write_text(json.dumps({"bindings": [], "warnings": [warning]}), encoding="utf-8") + args = argparse.Namespace(bindings="bindings.json", mode="check", provider=None, project=None) + + assert policy_apply.run(args, logging.getLogger("test.fw2.policy")) == 0 + + shown = _warning_events(caplog, "policy_bindings_warning") + assert [e["warning"] for e in shown] == [warning] + + +def _cli(tmp_path: Path, *argv: str) -> subprocess.CompletedProcess: + env = {**os.environ, "PYTHONPATH": str(_REPO), "NO_COLOR": "1"} + return subprocess.run( + [sys.executable, "-m", "fluid_build", *argv], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + + +@pytest.mark.integration +def test_the_cli_prints_the_warning_on_stderr(tmp_path): + """The real entry point, as the defect was measured: the warning reaches + stderr (stdout stays the command's own output) and the exit stays 0.""" + contract = _write_contract(tmp_path, _LAKEKEEPER_GRANT) + result = _cli(tmp_path, "policy", "compile", contract.name, "--out", "out/b.json") + + assert result.returncode == 0, result.stderr + assert _NO_TABLE_GRANT in result.stderr + assert _NO_TABLE_GRANT not in result.stdout + + +@pytest.mark.integration +def test_the_cli_fails_on_a_compiler_crash(tmp_path): + """A grant that is a bare string crashes ``compile_policy``; that used to + write ``{"bindings": {"bindings": []}}`` and exit 0.""" + doc = json.loads(json.dumps(_LAKEKEEPER_GRANT)) + doc["accessPolicy"]["grants"] = ["group:analysts@example.com"] + contract = _write_contract(tmp_path, doc) + result = _cli(tmp_path, "policy", "compile", contract.name, "--out", "out/b.json") + + assert result.returncode == 1 + assert "ERR_POLICY_COMPILER_CRASHED" in result.stdout + result.stderr + assert not (tmp_path / "out" / "b.json").exists() diff --git a/tests/iac/test_fw2_snowflake_volume_message.py b/tests/iac/test_fw2_snowflake_volume_message.py new file mode 100644 index 00000000..4f1ae0aa --- /dev/null +++ b/tests/iac/test_fw2_snowflake_volume_message.py @@ -0,0 +1,151 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Snowflake EXTERNAL VOLUME guard names both causes, and blames neither. + +JRN-707-6: the volume is keyed by the contract id, so state cannot say which +expose it was created for. A contract applied only by the current release, with +a Snowflake-managed expose (which gets the volume) beside a Lakekeeper one, +leaves the same volume in state when the managed expose is removed. The guard +still blocks that apply (what it detects is unchanged), but its message said +"forge-cli no longer creates an EXTERNAL VOLUME ..." and listed the Lakekeeper +expose as the table the volume was held for. It now names both possible causes +and keeps the ``tofu state rm`` remediation. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.iac.catalog_moves import ( + CatalogMoveError, + detect_catalog_moves, + guard_catalog_moves, +) +from fluid_build.iac.providers.aws import AwsIacPlugin +from fluid_build.iac.providers.snowflake import SnowflakeIacPlugin + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +SCHEMA = [{"name": "order_id", "type": "string"}] +VOLUME = "snowflake_external_volume.analytics_lake_vol_FLUID_ANALYTICS_LAKE_VOL" +SF_TABLE = "snowflake_table.analytics_lake_ANALYTICS_SALES_ORDERS" +WORKDIR = "/w/snowflake/analytics.lake" + + +def _sf_expose(catalog: Optional[str], expose_id: str, table: str) -> Dict[str, Any]: + location: Dict[str, Any] = { + "database": "ANALYTICS", + "schema": "SALES", + "table": table, + "warehouse": "s3://lake/warehouse", + "iam_role_arn": "arn:aws:iam::123456789012:role/snowflake-lake", + } + if catalog is not None: + location.update(catalog=catalog, uri="http://lakekeeper:8181/catalog") + return { + "exposeId": expose_id, + "binding": {"platform": "snowflake", "format": "iceberg", "location": location}, + "contract": {"schema": copy.deepcopy(SCHEMA)}, + } + + +def _contract(*exposes: Dict[str, Any]) -> Dict[str, Any]: + return {"id": "analytics.lake", "name": "Lake", "exposes": list(exposes)} + + +#: After the change: the managed expose is gone, the Lakekeeper one stays. +_AFTER = _contract(_sf_expose("lakekeeper", "orders_iceberg", "ORDERS")) + + +def _blocked(contract: Dict[str, Any], state) -> CatalogMoveError: + with pytest.raises(CatalogMoveError) as excinfo: + guard_catalog_moves(SnowflakeIacPlugin(), contract, state, workdir=WORKDIR) + return excinfo.value + + +def test_the_current_release_s_own_volume_is_still_blocked(): + """Detection is unchanged: the volume a removed managed expose used is + flagged, as it was before the message changed.""" + plugin = SnowflakeIacPlugin() + before = _contract( + _sf_expose(None, "orders_managed", "ORDERS_MANAGED"), + _sf_expose("lakekeeper", "orders_iceberg", "ORDERS"), + ) + emitted = plugin.emit(before)["snowflake_external_volume"] + assert {f"snowflake_external_volume.{key}" for key in emitted} == {VOLUME} + + assert detect_catalog_moves(plugin, _AFTER, [SF_TABLE, VOLUME]) == (VOLUME,) + exc = _blocked(_AFTER, [SF_TABLE, VOLUME]) + assert exc.addresses == (VOLUME,) + assert exc.remediation == (f"tofu -chdir={WORKDIR} state rm {VOLUME}",) + + +def test_the_message_does_not_say_an_earlier_release_created_it(): + message = str(_blocked(_AFTER, [SF_TABLE, VOLUME])) + + assert "no longer creates" not in message + assert "an earlier dbt run" not in message + assert "for Iceberg table(s) that now live in another catalog" not in message + assert ( + "holds 1 Snowflake EXTERNAL VOLUME(s) that this contract's configuration no longer " + "declares:\n\n " + VOLUME + ) in message + + +def test_the_message_names_both_causes_before_the_exposes(): + message = str(_blocked(_AFTER, [SF_TABLE, VOLUME])) + + causes = message.index("Possible causes:") + upgrade = message.index("applied by a forge-cli release that gave an Iceberg table") + removal = message.index("this change removed a Snowflake-managed Iceberg expose") + recatalog = message.index("or moved one to another catalog") + expose = message.index(" exposes[orders_iceberg]: location.catalog lakekeeper") + assert "does not say which expose it was created for" in message + assert causes < upgrade < removal < recatalog < expose + + +def test_the_remediation_is_kept(): + message = str(_blocked(_AFTER, [SF_TABLE, VOLUME])) + + assert f" tofu -chdir={WORKDIR} state rm {VOLUME}" in message + assert "the resources stay in Snowflake" in message + assert "DROP EXTERNAL VOLUME" in message + + +def test_the_aws_message_still_attributes_the_glue_resources(): + """AWS has per-expose evidence (the Glue table), so its message is as before.""" + location = {"database": "sales", "table": "orders", "bucket": "lake", "path": "orders/"} + location["catalog"] = "lakekeeper" + contract = _contract( + { + "exposeId": "orders", + "binding": {"platform": "aws", "format": "iceberg", "location": location}, + "contract": {"schema": copy.deepcopy(SCHEMA)}, + } + ) + db = "aws_glue_catalog_database.analytics_lake_sales" + table = "aws_glue_catalog_table.analytics_lake_sales_orders" + with pytest.raises(CatalogMoveError) as excinfo: + guard_catalog_moves(AwsIacPlugin(), contract, [db, table]) + + assert str(excinfo.value).startswith( + "iceberg catalog move blocked — this contract's OpenTofu state holds 2 Glue catalog " + "resource(s) for Iceberg table(s) that now live in another catalog:\n\n" + f" exposes[orders]: location.catalog lakekeeper\n\n {db}\n {table}\n\n" + "forge-cli no longer creates a Glue table" + ) From 23d0639b0d96d6430dde4ac8af708d15787d716a Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:15:15 +0200 Subject: [PATCH 02/19] fix(policy): state what the policy compile and apply comments mean, without counts --- fluid_build/cli/policy_apply.py | 4 ++-- fluid_build/cli/policy_compile.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fluid_build/cli/policy_apply.py b/fluid_build/cli/policy_apply.py index 330f5444..97d21174 100644 --- a/fluid_build/cli/policy_apply.py +++ b/fluid_build/cli/policy_apply.py @@ -89,8 +89,8 @@ def run(args, logger: logging.Logger) -> int: # The compiler's warnings, echoed here too: a grant that compiled to # no binding (see ``fluid policy compile``) is not enforced by this - # step either, and the run that applies is often not the one that - # compiled. A contract with no grants leaves nothing unenforced. + # step either, and apply can run in a different job from compile. A + # contract with no grants leaves nothing unenforced. from fluid_build.policy.compiler import NO_GRANTS stored = data.get("warnings") if isinstance(data, dict) else None diff --git a/fluid_build/cli/policy_compile.py b/fluid_build/cli/policy_compile.py index 77ecc02b..b5883422 100644 --- a/fluid_build/cli/policy_compile.py +++ b/fluid_build/cli/policy_compile.py @@ -50,7 +50,7 @@ def register(subparsers: argparse._SubParsersAction): #: The values ``compile_policy`` reads as a mapping or a list. A wrong type at -#: one of these paths is a value the compiler cannot read. Any other schema +#: one of these paths is one the compiler can fail on. Any other schema #: error (an extra key on a grant, a wrong type under ``metadata``) is not #: named: it did not make the compiler fail. _COMPILER_READ_PATHS = re.compile( @@ -105,8 +105,8 @@ def run(args, logger: logging.Logger) -> int: bindings, warnings = compile_policy(c) except Exception as e: - # Fail closed. This used to write ``{"bindings": []}`` and exit 0, - # so a crash read as "no grants to enforce" to the next stage. + # Fail closed. This used to write an empty bindings list and exit + # 0, so a crash read as "no grants to enforce" to the next stage. # Nothing is written: a bindings file from an earlier run stays as # it was, and the non-zero exit stops the pipeline before apply. logger.debug(traceback.format_exc()) From ce0aa20b4ad37930c9f75dffe38cf6e000de2594 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:47:27 +0200 Subject: [PATCH 03/19] fix(iceberg): a derived streaming sink writes the exposes its build's outputs pick Every Kafka Connect and embedded Debezium Iceberg sink resolved its table from exposes[0], so a second CDC build whose outputs named another Iceberg expose streamed into the first expose's table while validate, the preflight and the run all passed. A sink config forge-cli derives now resolves its exposes from build.outputs through resolve_iceberg_sink_exposes: - Kafka Connect derives one iceberg.tables entry: exactly one Iceberg sink expose in outputs, or no outputs and exactly one in the contract. Anything else is a hard validate error. - Embedded Debezium Server derives one table-namespace and one catalog: the exposes outputs pick (all of them with no outputs) must share one database and resolve to one catalog, else a hard error. Validate, the runner preflight, both runners and the Kafka Connect late-arrival target read iceberg_sink_target, which applies this join to derived configs. A hand-written config, or an override that sets iceberg.tables or table-namespace, is checked against the first Iceberg sink expose as before. --- fluid_build/build_runners/debezium/runner.py | 16 +- .../kafka_connect/iceberg_sink_validation.py | 67 +++- .../build_runners/kafka_connect/runner.py | 23 +- fluid_build/providers/_iceberg_catalog.py | 151 +++++++- tests/build_runners/test_fw2_sink_expose.py | 363 ++++++++++++++++++ .../test_iceberg_sink_validation.py | 18 +- tests/providers/test_fw2_sink_expose.py | 216 +++++++++++ 7 files changed, 825 insertions(+), 29 deletions(-) create mode 100644 tests/build_runners/test_fw2_sink_expose.py create mode 100644 tests/providers/test_fw2_sink_expose.py diff --git a/fluid_build/build_runners/debezium/runner.py b/fluid_build/build_runners/debezium/runner.py index 295f3fd1..3d96e6e8 100644 --- a/fluid_build/build_runners/debezium/runner.py +++ b/fluid_build/build_runners/debezium/runner.py @@ -547,16 +547,18 @@ def _execute_debezium_server( if preflight_error: return _failed(ctx, started_at, t_start, preflight_error) if sink_plan is not None and sink_plan.derives: - from ...providers._iceberg_catalog import ( - find_iceberg_expose_binding, - resolve_iceberg_catalog, - ) + from ...providers._iceberg_catalog import resolve_iceberg_catalog + from ..kafka_connect.iceberg_sink_validation import iceberg_sink_target from .iceberg_sink import emit_debezium_iceberg_sink_config - binding = find_iceberg_expose_binding(ctx.contract) - if binding is not None: + # The exposes the build's outputs name, as `fluid validate` resolves + # them. They share one database and one catalog, so the first one's + # binding gives the namespace and the catalog block (for Glue, whose + # warehouse is a per-table prefix, the first one's prefix). + targets, _error = iceberg_sink_target(ctx.contract, build, sink_plan) + if targets: resolved = resolve_iceberg_catalog( - binding, + targets[0].get("binding") or {}, contract=ctx.contract, sink=ctx.sink, account_ref=ctx.env.get("AWS_ACCOUNT_ID", ""), diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 4aff656f..83ac8193 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -88,6 +88,7 @@ iceberg_sink_exposes, known_catalog_kinds, resolve_iceberg_catalog, + resolve_iceberg_sink_exposes, ) LOG = logging.getLogger("fluid.acquire.iceberg_sink") @@ -151,6 +152,10 @@ class IcebergSinkPlan: overrides: Tuple[Tuple[str, Mapping[str, Any]], ...] #: ``(label, value)`` of the warehouse override the zero-drift check reads. warehouse_override: Tuple[str, Any] + #: The key that names the tables the sink writes (``iceberg.tables`` for + #: Kafka Connect, ``table-namespace`` for Debezium Server). An override map + #: that sets it names the tables itself instead of the derived config. + table_key: str = "" def iceberg_sink_plan(build: Mapping[str, Any]) -> Optional[IcebergSinkPlan]: @@ -191,6 +196,7 @@ def iceberg_sink_plan(build: Mapping[str, Any]) -> Optional[IcebergSinkPlan]: impl_key="catalog-impl", overrides=((label, config),), warehouse_override=(label, config.get("warehouse")), + table_key="table-namespace", ) if not declares_iceberg: @@ -217,9 +223,43 @@ def iceberg_sink_plan(build: Mapping[str, Any]) -> Optional[IcebergSinkPlan]: "iceberg_catalog_overrides", catalog_overrides.get("iceberg.catalog.warehouse"), ), + table_key="iceberg.tables", ) +def derives_table_identity(plan: IcebergSinkPlan) -> bool: + """Does forge-cli derive the tables ``plan``'s sink config writes? + + ``True`` when the runner derives the config and no override map sets + :attr:`IcebergSinkPlan.table_key`. Such a config writes the exposes the + build's outputs pick (:func:`iceberg_sink_target`). + """ + return plan.derives and not any(plan.table_key in m for _label, m in plan.overrides) + + +def iceberg_sink_target( + contract: Mapping[str, Any], + build: Mapping[str, Any], + plan: Optional[IcebergSinkPlan] = None, +) -> Tuple[Tuple[Mapping[str, Any], ...], Optional[str]]: + """The Iceberg sink exposes ``build``'s sink writes, or why there are none. + + THE build->expose join: ``fluid validate``, the runner preflight, both + streaming runners and the Kafka-Connect late-arrival target read it, so + they name the same tables. A config forge-cli derives + (:func:`derives_table_identity`) writes the exposes + :func:`resolve_iceberg_sink_exposes` joins from the build's outputs, by + the runtime's rule (one table for Kafka Connect, one namespace for + Debezium Server). Any other config names its own tables and is checked + against the first Iceberg sink expose. ``plan`` defaults to + :func:`iceberg_sink_plan` of ``build``. + """ + plan = plan if plan is not None else iceberg_sink_plan(build) + if plan is not None and derives_table_identity(plan): + return resolve_iceberg_sink_exposes(contract, build, namespace=plan.engine == "debezium") + return iceberg_sink_exposes(contract)[:1], None + + def validate_iceberg_sink(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: """Return (errors, warnings) for every Iceberg streaming-sink build.""" return _validate(contract) @@ -267,12 +307,13 @@ def _validate(contract: Mapping[str, Any], *, builds: Optional[Iterable[Any]] = runtime = iceberg_sink_plan(build) if runtime is None: continue # not a build that writes an Iceberg sink - _check_build(build, runtime, iceberg_exposes, expose_ids, errors, warnings) + _check_build(contract, build, runtime, iceberg_exposes, expose_ids, errors, warnings) return errors, warnings def _check_build( + contract: Mapping[str, Any], build: Mapping[str, Any], runtime: IcebergSinkPlan, iceberg_exposes: Sequence[Mapping[str, Any]], @@ -294,13 +335,19 @@ def _check_build( "the connector has no table identity to write to" ) return + # The exposes the runner writes, by the resolver the runners call. A + # derived config writes the exposes the build's outputs pick, and outputs + # that pick none it can write are an error; a config that names its own + # tables is checked against the first Iceberg expose. HARD. + targets, join_error = iceberg_sink_target(contract, build, runtime) + if join_error: + errors.append(join_error) outputs = build.get("outputs") or [] - if outputs and not (set(outputs) & expose_ids): + if not derives_table_identity(runtime) and outputs and not (set(outputs) & expose_ids): warnings.append( f"iceberg sink (build {bid!r}) outputs {list(outputs)} don't reference the " f"Iceberg expose(s) {sorted(x for x in expose_ids if x)}; the join is implicit" ) - binding = iceberg_exposes[0].get("binding") or {} # 2. upsert is deferred to v2 (locked v1 decision) — gate, don't silently # append-only. HARD. @@ -318,7 +365,19 @@ def _check_build( "streamingSink.routeField" ) - _check_catalog(bid, binding, sink, runtime, errors, warnings) + # The catalog checks read each expose the sink writes. A Debezium Server + # sink writes several only when they resolve to one catalog, so a message + # an earlier expose produced is not repeated. + for index, target in enumerate(targets): + binding = target.get("binding") or {} + if index == 0: + _check_catalog(bid, binding, sink, runtime, errors, warnings) + continue + more_errors: List[str] = [] + more_warnings: List[str] = [] + _check_catalog(bid, binding, sink, runtime, more_errors, more_warnings) + errors.extend(m for m in more_errors if m not in errors) + warnings.extend(m for m in more_warnings if m not in warnings) def _check_catalog( diff --git a/fluid_build/build_runners/kafka_connect/runner.py b/fluid_build/build_runners/kafka_connect/runner.py index 29067489..911029cb 100644 --- a/fluid_build/build_runners/kafka_connect/runner.py +++ b/fluid_build/build_runners/kafka_connect/runner.py @@ -95,15 +95,20 @@ def resolve_sink_connector(format_or_platform: str, override: Optional[str] = No return SINK_CONNECTOR_CLASS["s3"] # safe default -def _find_iceberg_expose_binding(contract: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """The expose ``binding`` carrying the Iceberg-table identity for a sink. - - Delegates to the shared resolver helper so the Kafka-Connect and - Debezium-Server runners pick the SAME table (RFC zero-drift spine). +def _find_iceberg_expose_binding( + contract: Mapping[str, Any], build: Mapping[str, Any] +) -> Optional[Dict[str, Any]]: + """The expose ``binding`` carrying the Iceberg-table identity for ``build``'s sink. + + Delegates to the shared resolver so the Kafka-Connect and Debezium-Server + runners pick the SAME table (RFC zero-drift spine): the expose + ``iceberg_sink_target`` resolves, the one ``fluid validate`` checks. + ``None`` when it resolves none (the preflight refuses that build first). """ - from ...providers._iceberg_catalog import find_iceberg_expose_binding + from .iceberg_sink_validation import iceberg_sink_target - return find_iceberg_expose_binding(contract) + exposes, _error = iceberg_sink_target(contract, build) + return dict(exposes[0].get("binding") or {}) if exposes else None # ── REST client ──────────────────────────────────────────────────────── @@ -338,7 +343,7 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: # only; nothing provisions the side table yet (RFC §6.7 / spike §14). late_arrival_target = connector_name_for_topic if str(ctx.sink.format or "").lower() == "iceberg": - _ib = _find_iceberg_expose_binding(ctx.contract) or {} + _ib = _find_iceberg_expose_binding(ctx.contract, build) or {} _loc = _ib.get("location") or {} if _loc.get("database") and _loc.get("table"): late_arrival_target = f"{_loc['database']}.{_loc['table']}" @@ -370,7 +375,7 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: if preflight_error: return _failed(ctx, started_at, t_start, preflight_error) if sink_plan is not None and sink_plan.derives: - binding = _find_iceberg_expose_binding(ctx.contract) + binding = _find_iceberg_expose_binding(ctx.contract, build) if binding is not None: from ...providers._iceberg_catalog import resolve_iceberg_catalog from .iceberg_sink import emit_iceberg_sink_config diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 956bf1b9..c5c2819c 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -298,17 +298,156 @@ def iceberg_sink_exposes(contract: Mapping[str, Any]) -> Tuple[Mapping[str, Any] def find_iceberg_expose_binding(contract: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """The expose ``binding`` carrying the Iceberg-table identity for a sink. - - Shared by the Kafka-Connect and Debezium-Server runners so both resolve the - SAME table identity (the RFC zero-drift spine): the first of - :func:`iceberg_sink_exposes`. The validated build->expose join - (build.outputs / exposeId) lands with the plan-time validator (RFC §6.8 #5). + """The binding of the first of :func:`iceberg_sink_exposes`, or ``None``. + + A sink config that names its own tables (hand-written, or an override that + sets ``iceberg.tables`` / ``table-namespace``) is checked against this + expose's catalog. A sink config forge-cli derives reads the exposes + :func:`resolve_iceberg_sink_exposes` joins from the build's outputs; the + streaming runners select between the two through + ``iceberg_sink_validation.iceberg_sink_target``. """ exposes = iceberg_sink_exposes(contract) return dict(exposes[0].get("binding") or {}) if exposes else None +def _expose_id(expose: Mapping[str, Any]) -> Any: + return expose.get("exposeId") or expose.get("id") + + +def _expose_database(expose: Mapping[str, Any]) -> Any: + """The database leg of an expose's table, read as :func:`resolve_iceberg_catalog` reads it.""" + binding = expose.get("binding") or {} + return (binding.get("location") or {}).get("database") or binding.get("database") + + +def _described(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: + """``["orders (sales.orders)", ...]``: each expose's id and table, for the messages.""" + return [ + f"{_expose_id(e)} ({resolve_iceberg_catalog(e.get('binding') or {}).fq_table})" + for e in exposes + ] + + +def resolve_iceberg_sink_exposes( + contract: Mapping[str, Any], + build: Mapping[str, Any], + *, + namespace: bool = False, +) -> Tuple[Tuple[Mapping[str, Any], ...], Optional[str]]: + """The exposes a sink config forge-cli DERIVES for ``build`` writes, or why it cannot. + + The candidates are :func:`iceberg_sink_exposes`; the build's ``outputs`` + pick among them, and with no outputs the candidates are the contract's. + + * ``namespace=False`` (Kafka Connect): the derived config carries one + ``iceberg.tables`` entry, and the Apache Iceberg sink writes each record + to every table that lists (``SinkWriter.routeRecordStatically``), so it + writes one expose. Exactly one candidate in outputs, or no outputs and + exactly one in the contract; anything else is an error. + * ``namespace=True`` (embedded Debezium Server): the derived config carries + one ``table-namespace`` and one catalog, and the server writes each + captured table under that namespace (``DefaultIcebergTableMapper``). + The candidates selected must share one database and resolve to one + catalog; outputs that name none of them are an error. + + Returns ``(exposes, None)`` with at least one expose, ``((), error)``, or + ``((), None)`` when the contract has no Iceberg sink expose at all (the + validator reports that). + """ + candidates = iceberg_sink_exposes(contract) + if not candidates: + return (), None + head = f"iceberg sink (build {build.get('id', '?')!r})" + raw_outputs = build.get("outputs") or [] + outputs = [raw_outputs] if isinstance(raw_outputs, str) else list(raw_outputs) + selected = tuple(e for e in candidates if _expose_id(e) in outputs) if outputs else candidates + if outputs: + picked = f"its outputs {outputs} name {len(selected) or 'none'} of the" + else: + picked = f"it declares no outputs, and the contract has {len(candidates)}" + picked += f" Iceberg sink exposes {_described(candidates)}" + if outputs: + writes = f"the exposes its outputs {outputs} name, {_described(selected)}," + else: + writes = f"the contract's Iceberg sink exposes, {_described(selected)}," + + if not namespace: + if len(selected) == 1: + return selected, None + return (), ( + f"{head}: {picked}; a derived Kafka Connect sink writes one expose (one " + "iceberg.tables entry). List exactly one of them in the build's outputs, split " + "the build into one build per expose, or hand-write the sink config " + "(properties.kafka-connect.sink_connector_config)" + ) + + remedy = "hand-write the sink config (properties.debezium.server.sink.config)" + if not selected: + return (), ( + f"{head}: {picked}; a derived Debezium Server sink writes the exposes the " + f"build's outputs name. List them in the build's outputs, or {remedy}" + ) + databases = sorted({str(_expose_database(e)) for e in selected}) + if len(databases) > 1: + return (), ( + f"{head}: a derived Debezium Server sink writes every captured table under one " + f"table-namespace, but {writes} sit in the databases {databases}. Give them one " + f"binding.location.database, split the build into one build per database, or {remedy}" + ) + differs = _catalog_differences(selected) + if differs: + return (), ( + f"{head}: a derived Debezium Server sink writes through one catalog, but {writes} " + f"resolve to different catalogs (their {', '.join(differs)} differ). Give them one " + f"catalog, split the build into one build per catalog, or {remedy}" + ) + return selected, None + + +#: The binding setting each :class:`ResolvedIcebergCatalog` catalog field is +#: resolved from, for the messages. +_CATALOG_FIELD_SOURCES: Mapping[str, str] = MappingProxyType( + { + "kind": "catalog", + "catalog_type": "catalog", + "catalog_impl": "catalog", + "uri": "uri", + "warehouse": "warehouse", + "io_impl": "warehouse", + "region": "region", + "extra_catalog_props": "catalog properties", + } +) + + +def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: + """The catalog settings on which ``exposes`` resolve differently, or ``[]``. + + Compares every :class:`ResolvedIcebergCatalog` field except the table + identity and the per-table write defaults. A Glue warehouse is per table + too (``get_iceberg_warehouse`` derives ``s3://{bucket}/{database}/{table}/``), + while a REST catalog's warehouse is the catalog's name. + """ + from dataclasses import fields as dataclass_fields + + resolved = [resolve_iceberg_catalog(e.get("binding") or {}) for e in exposes] + if len(resolved) < 2: + return [] + per_table = {"fq_table", "id_columns", "partition_by"} + if all(r.catalog_impl == GLUE_CATALOG_IMPL for r in resolved): + per_table.add("warehouse") + differs: list[str] = [] + for f in dataclass_fields(ResolvedIcebergCatalog): + if f.name in per_table: + continue + if any(getattr(r, f.name) != getattr(resolved[0], f.name) for r in resolved[1:]): + source = _CATALOG_FIELD_SOURCES.get(f.name, f.name) + if source not in differs: + differs.append(source) + return differs + + #: Object-store URI schemes, and the Iceberg ``FileIO`` each one needs. THE one #: scheme list: the resolver picks a FileIO from it and the sink validator #: refuses a name-only warehouse that carries one. diff --git a/tests/build_runners/test_fw2_sink_expose.py b/tests/build_runners/test_fw2_sink_expose.py new file mode 100644 index 00000000..3595b167 --- /dev/null +++ b/tests/build_runners/test_fw2_sink_expose.py @@ -0,0 +1,363 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LOGIC-1: a derived streaming Iceberg sink writes the exposes its build's outputs pick. + +Two CDC builds feeding two Iceberg exposes is the ordinary shape. Every sink +used to write ``exposes[0]``, so the ``refunds`` build's connector carried +``iceberg.tables=sales.orders`` while ``fluid validate``, the preflight and the +run all passed. A sink config forge-cli derives now resolves its exposes from +``build.outputs`` through one resolver that ``fluid validate``, the preflight, +both runners and the late-arrival target read; a join it cannot resolve is a +HARD error that stops the run before any Connect REST call or config write. A +hand-written config, or an override that names the tables itself, behaves as +before. +""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.api.runner import RunState +from fluid_build.build_runners._acquisition_common import build_acquisition_run_context +from fluid_build.build_runners.debezium.runner import DebeziumRunner +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.build_runners.kafka_connect.runner import execute_kafka_connect_build + +pytestmark = [pytest.mark.unit] + +_SOURCE = { + "kind": "postgres", + "connection": {"host": "db", "port": 5432, "database": "app", "user": "u", "password": "p"}, + "mode": "incremental_append", + "watermark": {"strategy": "high_water_mark", "allowedLateness": "PT5M"}, +} + +_LAKEKEEPER = {"catalog": "lakekeeper", "uri": "http://lakekeeper:8181/catalog", "warehouse": "wh"} + + +def _expose( + eid: str, database: str, table: str, *, fmt: str = "iceberg", **location: Any +) -> Dict[str, Any]: + return { + "exposeId": eid, + "kind": "table", + "binding": { + "platform": "aws", + "format": fmt, + "location": { + "database": database, + "table": table, + "bucket": "lake", + "region": "us-east-1", + **location, + }, + }, + "contract": {"schema": []}, + } + + +def _kc_build(bid: str, outputs: Optional[List[str]], stream: str, **kc: Any) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": bid, + "pattern": "acquisition", + "engine": "kafka-connect", + "capabilities": ["streaming"], + "properties": { + "source": {**copy.deepcopy(_SOURCE), "streams": [stream]}, + "sink": {"format": "iceberg"}, + "kafka-connect": { + "deployment": {"server_url": "http://kafka-connect.test:8083"}, + "connector_name": f"src-{bid}", + **kc, + }, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _dbz_build(outputs: Optional[List[str]], **server_sink: Any) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": "ingest", + "pattern": "acquisition", + "engine": "debezium", + "capabilities": ["cdc", "streaming"], + "properties": { + "source": { + **copy.deepcopy(_SOURCE), + "mode": "cdc", + "streams": ["public.orders", "public.refunds"], + }, + "sink": {"format": "iceberg"}, + "debezium": {"deployment": {"mode": "embedded"}, "server": {"sink": server_sink}}, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _contract(exposes: List[Dict[str, Any]], builds: List[Dict[str, Any]]) -> Dict[str, Any]: + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.cdc", + "name": "Sales CDC", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "x@y.z"}}, + # copies: a test edits its contract, and the module-level exposes are shared + "exposes": copy.deepcopy(exposes), + "builds": copy.deepcopy(builds), + } + + +ORDERS = _expose("orders", "sales", "orders") +REFUNDS = _expose("refunds", "sales", "refunds") +FIN_REFUNDS = _expose("refunds", "finance", "refunds") +RAW = _expose("raw", "sales", "raw", fmt="parquet") + + +def _two_builds() -> Dict[str, Any]: + return _contract( + [ORDERS, FIN_REFUNDS], + [ + _kc_build("ingest_orders", ["orders"], "public.orders"), + _kc_build("ingest_refunds", ["refunds"], "public.refunds"), + ], + ) + + +def _run_embedded(contract: Dict[str, Any], tmp_path: Path): + ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + return DebeziumRunner().run(ctx) + + +def _props_path(contract: Dict[str, Any], tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" + + +# ── the defect: each derived sink writes ITS build's expose ───────────────── + + +def test_two_builds_validate_clean(): + assert validate_iceberg_sink(_two_builds()) == ([], []) + + +@pytest.mark.parametrize( + "index, table", + [(0, "sales.orders"), (1, "finance.refunds")], + ids=["first-build", "second-build"], +) +def test_kafka_connect_sink_and_late_arrival_target_follow_the_builds_expose( + kafka_connect_mock, tmp_path: Path, index, table +): + contract = _two_builds() + build = contract["builds"][index] + assert execute_kafka_connect_build(build, contract, tmp_path, dry_run=False) == 0 + name = build["properties"]["kafka-connect"]["connector_name"] + sink = kafka_connect_mock.connectors[f"{name}-sink"]["config"] + assert sink["iceberg.tables"] == table + source = kafka_connect_mock.connectors[name]["config"] + # the late-events side table sits beside the same target + assert source["fluid.late_arrival.side_output_table"] == f"{table}__late_events" + + +def test_embedded_debezium_namespace_follows_the_builds_expose(tmp_path: Path): + contract = _contract([ORDERS, FIN_REFUNDS], [_dbz_build(["refunds"])]) + assert validate_iceberg_sink(contract) == ([], []) + _run_embedded(contract, tmp_path) # no server binary: fails after writing the config + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=finance" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/finance/refunds/" in text + assert "table-namespace=sales" not in text + + +def test_validate_checks_the_catalog_of_the_expose_the_build_writes(): + # outputs pick the Lakekeeper expose, which has no uri; the first expose + # is a complete Glue binding, which is all the validator used to read. + no_uri = _expose("refunds", "sales", "refunds", catalog="lakekeeper", warehouse="wh") + contract = _contract([ORDERS, no_uri], [_kc_build("ingest", ["refunds"], "public.refunds")]) + errors, _ = validate_iceberg_sink(contract) + assert any("lakekeeper catalog requires binding.location.uri" in e for e in errors), errors + + +# ── Kafka Connect: a derived sink writes one expose; anything else is HARD ── + +_KC_REFUSED = [ + ([ORDERS, RAW], ["raw"], "outputs ['raw'] name none of the Iceberg sink exposes"), + ([ORDERS, REFUNDS], ["orders", "refunds"], "outputs ['orders', 'refunds'] name 2 of"), + ([ORDERS, REFUNDS], None, "declares no outputs, and the contract has 2 Iceberg sink exposes"), + ([ORDERS, REFUNDS], [], "declares no outputs, and the contract has 2 Iceberg sink exposes"), +] +_KC_IDS = ["outputs-name-non-iceberg", "outputs-name-two", "no-outputs-two", "empty-outputs-two"] + + +@pytest.mark.parametrize("exposes, outputs, message", _KC_REFUSED, ids=_KC_IDS) +def test_kafka_connect_validate_and_preflight_refuse(exposes, outputs, message): + contract = _contract(exposes, [_kc_build("ingest", outputs, "public.orders")]) + errors, warnings = validate_iceberg_sink(contract) + [error] = [e for e in errors if message in e] + # names the build, its candidate exposes, and the way out + assert "build 'ingest'" in error + assert "orders (sales.orders)" in error + assert "a derived Kafka Connect sink writes one expose" in error + assert "split the build" in error and "hand-write the sink config" in error + assert not any("join is implicit" in w for w in warnings) + preflight = iceberg_sink_preflight(contract, "ingest") + assert preflight is not None and message in preflight + + +@pytest.mark.parametrize("exposes, outputs, message", _KC_REFUSED, ids=_KC_IDS) +def test_kafka_connect_run_refuses_before_any_rest_call( + kafka_connect_mock, tmp_path: Path, exposes, outputs, message +): + contract = _contract(exposes, [_kc_build("ingest", outputs, "public.orders")]) + rc = execute_kafka_connect_build(contract["builds"][0], contract, tmp_path, dry_run=False) + assert rc != 0 + assert kafka_connect_mock.connectors == {} + + +# ── embedded Debezium: one namespace and one catalog ──────────────────────── + + +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) +def test_embedded_debezium_writes_one_namespace_for_several_exposes(tmp_path: Path, outputs): + # DefaultIcebergTableMapper.mapDestination writes each captured table + # under table-namespace, so two exposes in one database and one catalog + # are one derived sink. + contract = _contract([ORDERS, REFUNDS], [_dbz_build(outputs)]) + assert validate_iceberg_sink(contract) == ([], []) + _run_embedded(contract, tmp_path) # no server binary: fails after writing the config + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=sales" in text + + +_DBZ_REFUSED = [ + ([ORDERS, RAW], ["raw"], "outputs ['raw'] name none of the Iceberg sink exposes"), + ([ORDERS, FIN_REFUNDS], ["orders", "refunds"], "sit in the databases ['finance', 'sales']"), + ([ORDERS, FIN_REFUNDS], None, "sit in the databases ['finance', 'sales']"), + ( + [ORDERS, _expose("refunds", "sales", "refunds", **_LAKEKEEPER)], + ["orders", "refunds"], + "resolve to different catalogs (their catalog, warehouse, uri differ)", + ), + ( + [ORDERS, _expose("refunds", "sales", "refunds", region="eu-west-1")], + None, + "resolve to different catalogs (their region differ)", + ), +] +_DBZ_IDS = [ + "outputs-name-non-iceberg", + "two-databases", + "no-outputs-two-databases", + "two-catalog-kinds", + "no-outputs-two-regions", +] + + +@pytest.mark.parametrize("exposes, outputs, message", _DBZ_REFUSED, ids=_DBZ_IDS) +def test_embedded_debezium_validate_preflight_and_run_refuse( + tmp_path: Path, exposes, outputs, message +): + contract = _contract(exposes, [_dbz_build(outputs)]) + errors, _ = validate_iceberg_sink(contract) + assert any("build 'ingest'" in e and message in e for e in errors), errors + preflight = iceberg_sink_preflight(contract, "ingest") + assert preflight is not None and message in preflight + result = _run_embedded(contract, tmp_path) + assert result.state == RunState.FAILED + assert message in (result.error or "") + assert not _props_path(contract, tmp_path).exists() + + +# ── unchanged: single-expose contracts and configs that name their tables ─── + + +@pytest.mark.parametrize("outputs", [None, [], ["orders"]], ids=["absent", "empty", "named"]) +def test_single_expose_kafka_connect_contract_is_unchanged( + kafka_connect_mock, tmp_path: Path, outputs +): + contract = _contract([ORDERS], [_kc_build("ingest", outputs, "public.orders")]) + assert validate_iceberg_sink(contract) == ([], []) + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest-sink"]["config"] + assert sink["iceberg.tables"] == "sales.orders" + + +@pytest.mark.parametrize("outputs", [None, ["orders"]], ids=["absent", "named"]) +def test_single_expose_embedded_debezium_contract_is_unchanged(tmp_path: Path, outputs): + contract = _contract([ORDERS], [_dbz_build(outputs)]) + assert validate_iceberg_sink(contract) == ([], []) + _run_embedded(contract, tmp_path) + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=sales" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/orders/" in text + + +@pytest.mark.parametrize( + "kc", + [ + {"iceberg_catalog_overrides": {"iceberg.tables": "sales.refunds"}}, + { + "iceberg_sink_enabled": True, + "sink_connector_config": {"iceberg.tables": "sales.refunds"}, + }, + {"sink_connector_config": {"iceberg.tables": "sales.refunds"}}, + ], + ids=["override", "hand-written-over-derived", "hand-written"], +) +@pytest.mark.parametrize("outputs", [None, ["orders", "refunds"]], ids=["absent", "two"]) +def test_kafka_connect_config_naming_its_tables_is_unchanged( + kafka_connect_mock, tmp_path: Path, kc, outputs +): + contract = _contract([ORDERS, REFUNDS], [_kc_build("ingest", outputs, "public.refunds", **kc)]) + assert validate_iceberg_sink(contract) == ([], []) + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest-sink"]["config"] + assert sink["iceberg.tables"] == "sales.refunds" + # the late-arrival target is still the first Iceberg expose, as before + source = kafka_connect_mock.connectors["src-ingest"]["config"] + assert source["fluid.late_arrival.side_output_table"] == "sales.orders__late_events" + + +def test_kafka_connect_hand_written_outputs_naming_no_expose_still_warn(): + kc = {"sink_connector_config": {"iceberg.tables": "sales.orders"}} + contract = _contract([ORDERS, RAW], [_kc_build("ingest", ["raw"], "public.orders", **kc)]) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert any("join is implicit" in w for w in warnings) + + +@pytest.mark.parametrize( + "server_sink", + [ + {"iceberg_sink_enabled": True, "config": {"table-namespace": "finance"}}, + {"config": {"catalog-impl": "org.apache.iceberg.aws.glue.GlueCatalog"}}, + ], + ids=["namespace-override", "hand-written"], +) +def test_embedded_debezium_config_naming_its_namespace_is_unchanged(tmp_path: Path, server_sink): + # two databases would refuse a derived namespace; these name their own + contract = _contract([ORDERS, FIN_REFUNDS], [_dbz_build(None, **server_sink)]) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None diff --git a/tests/build_runners/test_iceberg_sink_validation.py b/tests/build_runners/test_iceberg_sink_validation.py index 48b8b504..8cfbe3b8 100644 --- a/tests/build_runners/test_iceberg_sink_validation.py +++ b/tests/build_runners/test_iceberg_sink_validation.py @@ -90,9 +90,21 @@ def test_iceberg_sink_without_iceberg_expose_errors(): assert any("no expose with binding.format=iceberg" in e for e in errs) -def test_outputs_not_referencing_iceberg_expose_warns(): - warns = _warns(_contract(outputs=["something_else"])) - assert any("don't reference the Iceberg expose" in w for w in warns) +def test_outputs_not_referencing_iceberg_expose_errors_for_a_derived_sink(): + # A derived sink writes the expose its outputs name, so outputs that name + # none leave it no table; this was a warning, and the sink wrote exposes[0]. + contract = _contract(outputs=["something_else"]) + assert any( + "name none of the Iceberg sink exposes ['events (s.o)']" in e for e in _errs(contract) + ) + assert _warns(contract) == [] + + +def test_outputs_not_referencing_iceberg_expose_warns_for_a_hand_written_sink(): + hand_written = {"sink_connector_config": {"iceberg.tables": "s.o"}} + contract = _contract(outputs=["something_else"], kc=hand_written) + assert _errs(contract) == [] + assert any("don't reference the Iceberg expose" in w for w in _warns(contract)) def test_iceberg_table_alias_binding_counts_as_iceberg_expose(): diff --git a/tests/providers/test_fw2_sink_expose.py b/tests/providers/test_fw2_sink_expose.py new file mode 100644 index 00000000..495b875e --- /dev/null +++ b/tests/providers/test_fw2_sink_expose.py @@ -0,0 +1,216 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LOGIC-1: the build->expose join for a sink config forge-cli derives. + +``resolve_iceberg_sink_exposes`` picks the Iceberg sink exposes from the +build's outputs. Kafka Connect derives one ``iceberg.tables`` entry, so it +resolves exactly one expose; embedded Debezium Server derives one +``table-namespace`` and one catalog, so it resolves several when they share a +database and resolve to one catalog. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.providers._iceberg_catalog import ( + find_iceberg_expose_binding, + resolve_iceberg_sink_exposes, +) + +pytestmark = [pytest.mark.unit] + + +def _expose(eid: str, database: str, table: str, **location: Any) -> Dict[str, Any]: + return { + "exposeId": eid, + "binding": { + "platform": "aws", + "format": "iceberg", + "location": {"database": database, "table": table, "bucket": "lake", **location}, + }, + } + + +def _ids(exposes) -> List[str]: + return [e["exposeId"] for e in exposes] + + +def _build(outputs: Optional[Any]) -> Dict[str, Any]: + build: Dict[str, Any] = {"id": "ingest"} + if outputs is not None: + build["outputs"] = outputs + return build + + +ORDERS = _expose("orders", "sales", "orders") +REFUNDS = _expose("refunds", "sales", "refunds") +CONFLUENT = { + "exposeId": "managed", + "binding": {"platform": "confluent", "format": "iceberg", "location": {"table": "m"}}, +} +PARQUET = {"exposeId": "raw", "binding": {"platform": "aws", "format": "parquet"}} + + +# ── Kafka Connect (one table) ─────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "exposes, outputs, expected", + [ + ([ORDERS, REFUNDS], ["refunds"], ["refunds"]), + ([ORDERS, REFUNDS], ["orders"], ["orders"]), + ([ORDERS, REFUNDS, PARQUET], ["raw", "refunds"], ["refunds"]), + ([ORDERS], None, ["orders"]), + ([ORDERS], [], ["orders"]), + ([ORDERS], "orders", ["orders"]), + ([CONFLUENT, ORDERS], None, ["orders"]), + ], + ids=[ + "second-expose", + "first-expose", + "skips-non-iceberg-output", + "single-no-outputs", + "single-empty-outputs", + "outputs-as-a-string", + "confluent-is-not-a-candidate", + ], +) +def test_one_table_resolves(exposes, outputs, expected): + target, error = resolve_iceberg_sink_exposes({"exposes": exposes}, _build(outputs)) + assert error is None + assert _ids(target) == expected + + +@pytest.mark.parametrize( + "exposes, outputs, message", + [ + ([ORDERS, PARQUET], ["raw"], "outputs ['raw'] name none of the Iceberg sink exposes"), + ([ORDERS, CONFLUENT], ["managed"], "outputs ['managed'] name none"), + ([ORDERS, REFUNDS], ["orders", "refunds"], "outputs ['orders', 'refunds'] name 2 of"), + ([ORDERS, REFUNDS], None, "declares no outputs, and the contract has 2"), + ], + ids=["non-iceberg", "confluent-managed", "two-outputs", "no-outputs"], +) +def test_one_table_refuses(exposes, outputs, message): + target, error = resolve_iceberg_sink_exposes({"exposes": exposes}, _build(outputs)) + assert target == () + assert error is not None and message in error + assert error.startswith("iceberg sink (build 'ingest'): ") + assert "['orders (sales.orders)'" in error + assert "a derived Kafka Connect sink writes one expose" in error + + +def test_no_iceberg_sink_expose_is_left_to_the_validator(): + assert resolve_iceberg_sink_exposes({"exposes": [PARQUET]}, _build(["raw"])) == ((), None) + assert resolve_iceberg_sink_exposes({}, _build(None), namespace=True) == ((), None) + + +# ── embedded Debezium Server (one namespace, one catalog) ─────────────────── + + +@pytest.mark.parametrize( + "exposes, outputs, expected", + [ + ([ORDERS, REFUNDS], ["orders", "refunds"], ["orders", "refunds"]), + ([ORDERS, REFUNDS], None, ["orders", "refunds"]), + ([ORDERS, REFUNDS], ["refunds"], ["refunds"]), + ([ORDERS, _expose("refunds", "finance", "refunds")], ["refunds"], ["refunds"]), + ], + ids=["both-outputs", "no-outputs", "one-output", "one-output-other-database"], +) +def test_namespace_resolves(exposes, outputs, expected): + target, error = resolve_iceberg_sink_exposes( + {"exposes": exposes}, _build(outputs), namespace=True + ) + assert error is None + assert _ids(target) == expected + + +def test_namespace_glue_warehouse_is_per_table_not_a_catalog_difference(): + # get_iceberg_warehouse derives s3://lake/sales// for each one + target, error = resolve_iceberg_sink_exposes( + {"exposes": [ORDERS, REFUNDS]}, _build(None), namespace=True + ) + assert error is None and len(target) == 2 + + +_LK = {"uri": "http://lakekeeper:8181/catalog", "warehouse": "wh"} + + +@pytest.mark.parametrize( + "refunds, message", + [ + (_expose("refunds", "finance", "refunds"), "sit in the databases ['finance', 'sales']"), + ( + _expose("refunds", "sales", "refunds", catalog="lakekeeper", **_LK), + "(their catalog, warehouse, uri differ)", + ), + (_expose("refunds", "sales", "refunds", region="eu-west-1"), "(their region differ)"), + ], + ids=["two-databases", "two-catalog-kinds", "two-regions"], +) +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["outputs", "no-outputs"]) +def test_namespace_refuses(refunds, message, outputs): + target, error = resolve_iceberg_sink_exposes( + {"exposes": [ORDERS, refunds]}, _build(outputs), namespace=True + ) + assert target == () + assert error is not None and message in error + assert "a derived Debezium Server sink" in error + assert "hand-write the sink config (properties.debezium.server.sink.config)" in error + + +@pytest.mark.parametrize( + "location, message", + [ + ({"warehouse": "other"}, "(their warehouse differ)"), + ({"uri": "http://other:8181/catalog"}, "(their uri differ)"), + ({"catalog": "polaris"}, "(their catalog differ)"), + ], + ids=["warehouse", "uri", "kind"], +) +def test_namespace_rest_catalogs_must_match(location, message): + orders = _expose("orders", "sales", "orders", catalog="lakekeeper", **_LK) + refunds = _expose("refunds", "sales", "refunds", **{"catalog": "lakekeeper", **_LK, **location}) + contract = {"exposes": [orders, refunds]} + target, error = resolve_iceberg_sink_exposes(contract, _build(None), namespace=True) + assert target == () and error is not None and message in error + same = { + "exposes": [orders, _expose("refunds", "sales", "refunds", catalog="lakekeeper", **_LK)] + } + assert _ids(resolve_iceberg_sink_exposes(same, _build(None), namespace=True)[0]) == [ + "orders", + "refunds", + ] + + +def test_namespace_outputs_naming_none_are_refused(): + target, error = resolve_iceberg_sink_exposes( + {"exposes": [ORDERS, PARQUET]}, _build(["raw"]), namespace=True + ) + assert target == () + assert error is not None and "outputs ['raw'] name none of the Iceberg sink exposes" in error + + +# ── the first-expose helper a config that names its tables is checked against ─ + + +def test_find_iceberg_expose_binding_is_still_the_first_iceberg_sink_expose(): + assert find_iceberg_expose_binding({"exposes": [CONFLUENT, ORDERS, REFUNDS]}) == ( + ORDERS["binding"] + ) From 5d82e6875a88d584e1f955d4a5d6279c23976ef5 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:11:29 +0200 Subject: [PATCH 04/19] fix(iceberg): sink configs carry what dynamodb, jdbc and bigquery need to start; confluent defaults to glue CON-3: Iceberg's DynamoDbCatalog and JdbcCatalog refuse to initialise without a warehouse, but the kind table asked nothing (dynamodb) or only uri (jdbc) of them, and the resolver ignored location.bucket for them. Both now require a warehouse. The resolver takes location.warehouse, else derives one from an explicit location.bucket (and path, as the Glue helper builds it) with the platform scheme (aws s3://, gcp gs://), never from the account-derived fallback bucket. When none can be derived, validate errors and the runner preflight refuses. Both the Kafka Connect and the embedded Debezium configs emit the warehouse. CON-4: BigQueryMetastoreCatalog refuses to initialise without gcp.bigquery.project-id. location.project now maps to it (a missing project is a validate error and a preflight refusal), location.region maps to gcp.bigquery.location instead of the AWS-only client.region, and the gs:// storage URI dbt-bigquery and the GCP IaC use becomes the warehouse, with a validate warning when none can be derived. The Kafka Connect deriver now forwards the resolver's catalog-specific properties under the iceberg.catalog. prefix, as the Debezium deriver already did. CON-5: default_catalog_kind returns glue for platform confluent, the catalog the Tableflow IaC publishes an absent-catalog table to. Policy compile now grants the Glue table instead of dropping every grant with a false "cataloged in 'rest'" warning, and dbt-snowflake keeps the Glue linked-database type. --- .../build_runners/debezium/iceberg_sink.py | 7 +- .../kafka_connect/iceberg_sink.py | 6 + .../kafka_connect/iceberg_sink_validation.py | 84 +++++-- fluid_build/iac/providers/confluent.py | 9 +- fluid_build/providers/_iceberg_catalog.py | 88 ++++++- .../test_fw2_catalog_table_sink.py | 232 ++++++++++++++++++ .../test_iceberg_sink_validation.py | 9 +- .../test_fw2_confluent_default_catalog.py | 105 ++++++++ tests/providers/test_fw2_catalog_table.py | 184 ++++++++++++++ 9 files changed, 691 insertions(+), 33 deletions(-) create mode 100644 tests/build_runners/test_fw2_catalog_table_sink.py create mode 100644 tests/policy/test_fw2_confluent_default_catalog.py create mode 100644 tests/providers/test_fw2_catalog_table.py diff --git a/fluid_build/build_runners/debezium/iceberg_sink.py b/fluid_build/build_runners/debezium/iceberg_sink.py index 5d222378..54d45cc8 100644 --- a/fluid_build/build_runners/debezium/iceberg_sink.py +++ b/fluid_build/build_runners/debezium/iceberg_sink.py @@ -92,8 +92,11 @@ def emit_debezium_iceberg_sink_config( # GLOBAL (applies to all derived tables), not a per-table default. cfg["partition-by"] = ",".join(resolved.partition_by) - # Native Iceberg props (s3.* / gcs.* / adls.* / jdbc.* ...) pass straight - # through with the same prefix the runner adds. + # Native Iceberg props (s3.* / gcs.* / adls.* / jdbc.* / gcp.bigquery.* ...) + # pass straight through with the same prefix the runner adds: the server + # maps every ``debezium.sink.iceberg.*`` key into the catalog properties + # (memiiso/debezium-server-iceberg 1.2.0.Final IcebergConfig.java:18-21, + # IcebergChangeConsumer.java:100-102). for k, v in (resolved.extra_catalog_props or {}).items(): cfg[str(k)] = str(v) diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink.py b/fluid_build/build_runners/kafka_connect/iceberg_sink.py index 2066eb79..ef2e842d 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink.py @@ -111,6 +111,12 @@ def emit_iceberg_sink_config( cfg["iceberg.catalog.client.region"] = resolved.region if resolved.uri: cfg["iceberg.catalog.uri"] = resolved.uri + # Catalog-specific properties (``gcp.bigquery.project-id``...). The sink + # hands every ``iceberg.catalog.``-prefixed key to the catalog with the + # prefix stripped (apache-iceberg-1.10.0 kafka-connect/kafka-connect/src/ + # main/java/org/apache/iceberg/connect/IcebergSinkConfig.java:60, :254). + for k, v in (resolved.extra_catalog_props or {}).items(): + cfg[f"iceberg.catalog.{k}"] = str(v) # ── routing / write semantics ── if resolved.id_columns: diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 4aff656f..82bf5a68 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -43,8 +43,10 @@ * ``sink.catalog`` must agree with the expose's catalog, because dbt and the IaC read only the expose, and so must the catalog an override or a hand-written config selects (its ``type`` or ``catalog-impl``); -* every ``binding.location`` key in the row's ``sink_requires`` must be set - (Glue keeps its advisory region warning: the warehouse falls back); +* every ``binding.location`` key in the row's ``sink_requires`` must be set, + a warehouse counting when the deriver resolves one (DynamoDB and JDBC derive + it from an explicit bucket); Glue keeps its advisory region warning: the + warehouse falls back; * the runtime must ship the catalog's client (the stock Apache Iceberg Kafka Connect runtime has no Nessie client, and the published sink predates the ``bigquery`` catalog type); @@ -75,6 +77,8 @@ from typing import AbstractSet, Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from ...providers._iceberg_catalog import ( + BIGQUERY_PROJECT_ID, + BUCKET_WAREHOUSE_KINDS, DYNAMODB_CATALOG_IMPL, FAMILY_GLUE, FAMILY_REST, @@ -382,16 +386,17 @@ def _check_catalog( ) # 4. catalog tagged-union completeness, from the row's ``sink_requires``. - # HARD: forge can't derive a REST uri or a catalog name. Glue requires - # nothing (the warehouse falls back), so its region stays advisory. + # HARD: forge can't derive a REST uri or a catalog name. A warehouse is + # present when the deriver resolves one (DynamoDB and JDBC derive it + # from an explicit bucket). Glue requires nothing (the warehouse falls + # back), so its region stays advisory. + derived_warehouse = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse for key in info.sink_requires: - if not loc.get(key): - suffix = ( - " (the catalog name)" if key == "warehouse" and info.family == FAMILY_REST else "" - ) + present = derived_warehouse if key == "warehouse" else loc.get(key) + if not present: errors.append( f"iceberg sink (build {bid!r}): {kind} catalog requires " - f"binding.location.{key}{suffix}" + f"binding.location.{key}{_requires_hint(key, kind, info)}" ) if info.family == FAMILY_GLUE and not loc.get("region"): warnings.append( @@ -411,22 +416,73 @@ def _check_catalog( "the sink cannot load NessieCatalog" ) # ``type=bigquery`` (BigQueryMetastoreCatalog) is in Iceberg's CatalogUtil - # only from 1.10; the sink published on Confluent Hub is older, so a stock - # worker fails to load the catalog at connector start. Advisory: a runtime - # built from Iceberg >= 1.10 has it. + # only from 1.10 (apache-iceberg-1.10.0 CatalogUtil.java:78, :321-322; absent + # at apache-iceberg-1.9.2); the sink published on Confluent Hub is older, + # so a stock worker fails to load the catalog at connector start. + # Advisory: a runtime built from Iceberg >= 1.10 has it. if reaching == "bigquery" and runtime.engine == "kafka-connect": warnings.append( f"iceberg sink (build {bid!r}): the sink config sets " "iceberg.catalog.type=bigquery, which the published Apache Iceberg Kafka Connect " f"sink ({PUBLISHED_KAFKA_CONNECT_SINK_VERSION} on Confluent Hub) cannot load: " - "Iceberg's CatalogUtil gains the bigquery type in 1.10. Run a sink built from " - "Iceberg >= 1.10, or the connector fails at start" + "Iceberg's CatalogUtil gains the bigquery type in 1.10, so on that sink the " + "connector fails at start. Run a sink built from Iceberg >= 1.10" + ) + # BigQueryMetastoreCatalog reads the warehouse only to place a dataset it + # creates, or a table in a dataset with no default storage location URI + # (apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:147-160, :197-205, + # :298-303); an auto-creating sink creates both (IcebergWriterFactory.java + # :89, :122-131). Advisory: a dataset with a storage URI needs none. + if ( + reaching == "bigquery" + and kind == "bigquery" + and runtime.derives + and not derived_warehouse + and not _overrides_warehouse(runtime) + ): + warnings.append( + f"iceberg sink (build {bid!r}): no gs:// warehouse can be derived from " + "binding.location (a gs:// warehouse, or a bucket), so the derived bigquery " + "catalog config sets none. BigQueryMetastoreCatalog needs one to create a " + "dataset, or a table in a dataset with no default storage location URI" + + ( + "; the Debezium Server Iceberg sink also declares " + "debezium.sink.iceberg.warehouse with no default" + if runtime.engine == "debezium" + else "" + ) ) _check_warehouse_override(bid, binding, sink, kind, info, runtime, warnings) _check_selector_overrides(bid, kind, info, runtime, errors) +def _requires_hint(key: str, kind: str, info: CatalogKind) -> str: + """What a missing ``sink_requires`` key is to the sink, for check 4's error.""" + if key == "warehouse" and info.family == FAMILY_REST: + return " (the catalog name)" + if key == "warehouse" and kind in BUCKET_WAREHOUSE_KINDS: + return ( + " (an object-store location), or a binding.location.bucket on platform aws or " + f"gcp to derive it from; the {kind} catalog refuses to start without a warehouse" + ) + if key == "project" and kind == "bigquery": + return ( + f" (the sink's {BIGQUERY_PROJECT_ID}); the bigquery catalog refuses to start " + "without it" + ) + return "" + + +def _overrides_warehouse(runtime: IcebergSinkPlan) -> bool: + """Does an override map merged over the derived config set the warehouse?""" + # The warehouse key carries the same runtime prefix as the type key: + # ``iceberg.catalog.warehouse`` on Kafka Connect, bare ``warehouse`` on + # Debezium Server. + key = runtime.type_key[: -len("type")] + "warehouse" + return any(mapping.get(key) for _label, mapping in runtime.overrides) + + def _reaching_catalog_type(info: Any, runtime: IcebergSinkPlan) -> Optional[str]: """The catalog ``type`` the worker actually receives, or ``None``. diff --git a/fluid_build/iac/providers/confluent.py b/fluid_build/iac/providers/confluent.py index 9fbd0e30..9627605a 100644 --- a/fluid_build/iac/providers/confluent.py +++ b/fluid_build/iac/providers/confluent.py @@ -40,7 +40,7 @@ from typing import Any, Dict, Iterable, List, Mapping, Tuple -from ...providers._iceberg_catalog import binding_catalog_kind, canonical_catalog_kind +from ...providers._iceberg_catalog import binding_catalog_kind from ...providers.aws.util.warehouse import normalize_location from ..importer import ImportBlock from ..naming import safe_ident, tofu_ref @@ -84,11 +84,10 @@ def _non_glue_catalog(binding: Mapping[str, Any]) -> str: less claim on the name, in a catalog the contract never named. The emitter and ``validate_confluent_binding`` both read this, so the gate refuses exactly what the emitter declines to publish. An absent - ``catalog`` is Glue here, as it always was. + ``catalog`` is Glue: ``_iceberg_catalog.default_catalog_kind`` gives Glue for + ``platform: confluent``, so the policy compiler and dbt read the same + catalog this emitter publishes to. """ - loc = binding.get("location") or {} - if not canonical_catalog_kind(loc.get("catalog")): - return "" kind = binding_catalog_kind(binding) return "" if kind == "glue" else kind diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 956bf1b9..b8a72f9c 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -32,7 +32,7 @@ from typing import Any, Dict, Mapping, Optional, Tuple from ._sql_safety import validate_ident -from .aws.util.warehouse import get_iceberg_warehouse +from .aws.util.warehouse import bucket_uses_fallback, get_iceberg_warehouse, normalize_location # Apache Iceberg runtime class names (pinned to the connector surface validated # in the OSS spike — RFC §14). Bumping the Iceberg runtime may change these. @@ -138,12 +138,23 @@ def speaks_rest(self) -> bool: # Connect runtime does not bundle iceberg-nessie. CatalogKind("nessie", FAMILY_NATIVE, "nessie", None, "iceberg_rest", _REST_REQUIRES), # BigLake metastore. ``type=bigquery`` exists in Iceberg >= 1.10. - CatalogKind("bigquery", FAMILY_NATIVE, "bigquery", None, "iceberg_rest"), + # BigQueryMetastoreCatalog.initialize refuses to start without + # ``gcp.bigquery.project-id``, which the resolver maps from + # ``location.project`` (apache-iceberg-1.10.0 + # BigQueryMetastoreCatalog.java:84-87). + CatalogKind("bigquery", FAMILY_NATIVE, "bigquery", None, "iceberg_rest", ("project",)), CatalogKind("hive", FAMILY_NATIVE, "hive", None, None), - CatalogKind("jdbc", FAMILY_NATIVE, "jdbc", None, None, ("uri",)), + # JdbcCatalog.initialize refuses a missing uri and an empty warehouse + # (apache-iceberg-1.10.0 JdbcCatalog.java:114-119). The warehouse + # may be derived: see :data:`BUCKET_WAREHOUSE_KINDS`. + CatalogKind("jdbc", FAMILY_NATIVE, "jdbc", None, None, ("uri", "warehouse")), CatalogKind("hadoop", FAMILY_NATIVE, "hadoop", None, None, ("warehouse",)), # CatalogUtil has no ``dynamodb`` type: it is reached by impl only. - CatalogKind("dynamodb", FAMILY_NATIVE, None, DYNAMODB_CATALOG_IMPL, None), + # DynamoDbCatalog.initialize refuses an empty warehouse + # (apache-iceberg-1.10.0 DynamoDbCatalog.java:132-134). + CatalogKind( + "dynamodb", FAMILY_NATIVE, None, DYNAMODB_CATALOG_IMPL, None, ("warehouse",) + ), # Horizon over an EXTERNAL VOLUME; a streaming writer reaches it # through Snowflake's Iceberg REST endpoint. CatalogKind( @@ -158,6 +169,22 @@ def speaks_rest(self) -> bool: } ) +#: Kinds whose ``warehouse`` is an object-store location the resolver may +#: derive from an explicit ``location.bucket`` (and ``path``) when +#: ``location.warehouse`` is absent. Never from the account-derived fallback +#: bucket the Glue row uses: no IaC creates that bucket for a non-Glue table. +BUCKET_WAREHOUSE_KINDS = frozenset({"dynamodb", "jdbc"}) + +#: The object-store scheme a derived warehouse gets, by canonical platform. +_BUCKET_WAREHOUSE_SCHEMES: Mapping[str, str] = MappingProxyType({"aws": "s3", "gcp": "gs"}) + +#: BigQueryMetastoreCatalog's own property names (apache-iceberg-1.10.0 +#: bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/ +#: BigQueryMetastoreCatalog.java:62-63). Without a location the catalog uses +#: ``us`` (:68, :90). +BIGQUERY_PROJECT_ID = "gcp.bigquery.project-id" +BIGQUERY_LOCATION = "gcp.bigquery.location" + #: Accepted spellings for a canonical kind (after case and ``-``/``_`` folding). _CATALOG_ALIASES: Mapping[str, str] = MappingProxyType( {"iceberg-rest": "rest", "snowflake": "snowflake-managed"} @@ -200,8 +227,12 @@ def known_catalog_kinds() -> Tuple[str, ...]: def default_catalog_kind(binding: Mapping[str, Any]) -> str: - """The kind an Iceberg binding gets when it names none: Glue on AWS, - Snowflake-managed on Snowflake, a REST catalog anywhere else. + """The kind an Iceberg binding gets when it names none: Glue on AWS and on + Confluent, Snowflake-managed on Snowflake, a REST catalog anywhere else. + + Confluent is Glue because the Tableflow IaC publishes a table with no + ``catalog`` to AWS Glue (``iac/providers/confluent.py``), so the policy + compiler and dbt read the catalog the table is actually in. GCP is the one platform where an absent catalog means two things: the streaming sink has always written a REST catalog there, while dbt-bigquery @@ -213,7 +244,7 @@ def default_catalog_kind(binding: Mapping[str, Any]) -> str: from ..iac.provider_match import canonical_cloud cloud = canonical_cloud(binding.get("platform")) - if cloud == "aws": + if cloud in ("aws", "confluent"): return "glue" if cloud == "snowflake": return "snowflake-managed" @@ -380,7 +411,11 @@ def resolve_iceberg_catalog( ``account_ref`` feeds the warehouse bucket fallback on the Glue path (a concrete account id at connector-config time). REST catalogs take an explicit - ``uri`` + ``warehouse`` (catalog name) and don't use it. + ``uri`` + ``warehouse`` (catalog name) and don't use it. DynamoDB and JDBC + take ``location.warehouse``, else derive one from an explicit + ``location.bucket`` (:data:`BUCKET_WAREHOUSE_KINDS`); BigQuery takes the + ``gs://`` URI :func:`iceberg_storage_uri` gives, plus ``location.project`` + and ``location.region`` as its own catalog properties. """ loc = binding.get("location") or {} database = loc.get("database") or binding.get("database") @@ -411,6 +446,20 @@ def resolve_iceberg_catalog( # refuses it). The FileIO follows the WAREHOUSE scheme so GCS (gs://) and # ADLS (abfss://) work, not just S3 (RFC §6.3 — PR7's REST + GCP profiles). warehouse = loc.get("warehouse") or "" + region = loc.get("region") + extra: Dict[str, str] = {} + if kind == "bigquery": + # The gs:// storage dbt-bigquery's ``external_volume`` and the GCP IaC + # use, or none. ``client.region`` is an AWS client property, so the + # region goes to the catalog's own location key instead. + warehouse = iceberg_storage_uri(binding, scheme="gs") + if loc.get("project"): + extra[BIGQUERY_PROJECT_ID] = str(loc["project"]) + if region: + extra[BIGQUERY_LOCATION] = str(region) + region = None + elif not warehouse and kind in BUCKET_WAREHOUSE_KINDS: + warehouse = _bucket_warehouse(binding) return ResolvedIcebergCatalog( catalog_type=info.runtime_type or kind, warehouse=warehouse, @@ -418,13 +467,34 @@ def resolve_iceberg_catalog( catalog_impl=info.catalog_impl, uri=loc.get("uri"), io_impl=_io_impl_for_warehouse(warehouse), - region=loc.get("region"), + region=region, id_columns=id_columns, partition_by=partition_by, + extra_catalog_props=extra, kind=kind, ) +def _bucket_warehouse(binding: Mapping[str, Any]) -> str: + """The object-store warehouse an explicit ``location.bucket`` names, or ``""``. + + ``:///``, with the bucket and path normalised the way + the Glue row's :func:`get_iceberg_warehouse` normalises them (``path`` + defaults to ``/
/``). The scheme follows the platform: + ``s3`` on AWS, ``gs`` on GCP; any other platform derives nothing. A + missing or unresolved bucket derives nothing either, rather than the + Glue row's account-derived fallback bucket. + """ + from ..iac.provider_match import canonical_cloud + + loc = binding.get("location") or {} + scheme = _BUCKET_WAREHOUSE_SCHEMES.get(canonical_cloud(binding.get("platform"))) + if not scheme or bucket_uses_fallback(loc): + return "" + bucket, path = normalize_location(loc, account_ref="") + return f"{scheme}://{bucket}/{path}" + + # --------------------------------------------------------------------------- # Object-store warehouse URI # --------------------------------------------------------------------------- diff --git a/tests/build_runners/test_fw2_catalog_table_sink.py b/tests/build_runners/test_fw2_catalog_table_sink.py new file mode 100644 index 00000000..029a62a3 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_sink.py @@ -0,0 +1,232 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The derived sink configs carry what DynamoDB, JDBC and BigQuery need to start. + +CON-3: the derived dynamodb and jdbc configs had no warehouse, which both +catalogs refuse at initialise, yet validate and the preflight passed. CON-4: +the derived bigquery config had no ``gcp.bigquery.project-id`` (required at +initialise), no warehouse, and an AWS-only ``client.region``. Property names +are from apache/iceberg at apache-iceberg-1.10.0. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + +_JDBC_URI = "jdbc:postgresql://pg:5432/iceberg" + + +def _binding(catalog: str, platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc = {"catalog": catalog, "database": "streaming", "table": "orders", **location} + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _contract( + binding: Dict[str, Any], *, engine: str = "kafka-connect", props: Optional[Dict] = None +) -> Dict[str, Any]: + build_props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append"}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + build_props["debezium"] = props or { + "deployment": {"mode": "embedded"}, + "server": {"sink": {}}, + } + else: + build_props["kafka-connect"] = props or {} + return { + "id": "bronze.orders_stream", + "builds": [{"id": "ingest", "engine": engine, "properties": build_props}], + "exposes": [{"exposeId": "orders", "kind": "table", "binding": binding}], + } + + +def _kc(binding: Dict[str, Any]) -> Dict[str, str]: + resolved = resolve_iceberg_catalog(binding, account_ref="123456789012") + return emit_iceberg_sink_config(resolved, product_id="bronze.orders_stream", topics=["t"]) + + +def _dbz(binding: Dict[str, Any]) -> Dict[str, str]: + return emit_debezium_iceberg_sink_config( + resolve_iceberg_catalog(binding, account_ref="123456789012") + ) + + +# ── CON-3: dynamodb / jdbc ────────────────────────────────────────────────── + +_DERIVABLE = [ + _binding("dynamodb", bucket="acme-lake", region="eu-west-1"), + _binding("jdbc", bucket="acme-lake", region="eu-west-1", uri=_JDBC_URI), +] + + +@pytest.mark.parametrize("binding", _DERIVABLE, ids=["dynamodb", "jdbc"]) +def test_both_runtimes_emit_the_bucket_derived_warehouse(binding): + assert _kc(binding)["iceberg.catalog.warehouse"] == "s3://acme-lake/streaming/orders/" + assert _dbz(binding)["warehouse"] == "s3://acme-lake/streaming/orders/" + + +@pytest.mark.parametrize("binding", _DERIVABLE, ids=["dynamodb", "jdbc"]) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_derivable_warehouse_validates_clean(binding, engine): + contract = _contract(binding, engine=engine) + assert validate_iceberg_sink(contract)[0] == [] + assert iceberg_sink_preflight(contract, "ingest") is None + + +@pytest.mark.parametrize( + "binding", + [ + _binding("dynamodb", region="eu-west-1"), + _binding("jdbc", region="eu-west-1", uri=_JDBC_URI), + # no scheme to derive with off aws / gcp + _binding("dynamodb", "local", bucket="acme-lake"), + ], + ids=["dynamodb", "jdbc", "dynamodb-local"], +) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_no_derivable_warehouse_is_a_hard_error_and_the_preflight_refuses(binding, engine): + kind = binding["location"]["catalog"] + contract = _contract(binding, engine=engine) + errors = validate_iceberg_sink(contract)[0] + hit = [e for e in errors if f"{kind} catalog requires binding.location.warehouse" in e] + assert len(hit) == 1, errors + assert "binding.location.bucket" in hit[0] + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None and "binding.location.warehouse" in refusal + + +def test_jdbc_still_requires_its_uri(): + errors = validate_iceberg_sink(_contract(_binding("jdbc", bucket="acme-lake")))[0] + assert any("jdbc catalog requires binding.location.uri" in e for e in errors), errors + + +def test_an_explicit_warehouse_satisfies_dynamodb_off_aws(): + binding = _binding("dynamodb", "local", warehouse="s3://wh/root") + assert validate_iceberg_sink(_contract(binding))[0] == [] + assert _kc(binding)["iceberg.catalog.warehouse"] == "s3://wh/root" + + +# ── CON-4: bigquery ───────────────────────────────────────────────────────── + + +def _bq(**location: Any) -> Dict[str, Any]: + return _binding("bigquery", "gcp", **location) + + +_FULL_BQ = _bq(project="acme-proj", region="eu", bucket="acme-lake") + + +def test_kafka_connect_bigquery_config_carries_project_location_and_warehouse(): + cfg = _kc(_FULL_BQ) + assert cfg["iceberg.catalog.type"] == "bigquery" + assert cfg["iceberg.catalog.gcp.bigquery.project-id"] == "acme-proj" + assert cfg["iceberg.catalog.gcp.bigquery.location"] == "eu" + assert cfg["iceberg.catalog.warehouse"] == "gs://acme-lake" + assert "iceberg.catalog.client.region" not in cfg + + +def test_debezium_bigquery_config_carries_project_location_and_warehouse(): + cfg = _dbz(_FULL_BQ) + assert cfg["type"] == "bigquery" + assert cfg["gcp.bigquery.project-id"] == "acme-proj" + assert cfg["gcp.bigquery.location"] == "eu" + assert cfg["warehouse"] == "gs://acme-lake" + assert "client.region" not in cfg + + +def test_an_operator_override_still_wins_over_a_derived_bigquery_key(): + resolved = resolve_iceberg_catalog(_FULL_BQ) + cfg = emit_iceberg_sink_config( + resolved, + product_id="p", + topics=["t"], + kc_props={"iceberg_catalog_overrides": {"iceberg.catalog.gcp.bigquery.location": "us"}}, + ) + assert cfg["iceberg.catalog.gcp.bigquery.location"] == "us" + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_bigquery_without_project_is_a_hard_error_and_the_preflight_refuses(engine): + contract = _contract(_bq(bucket="acme-lake"), engine=engine) + errors = validate_iceberg_sink(contract)[0] + hit = [e for e in errors if "bigquery catalog requires binding.location.project" in e] + assert len(hit) == 1, errors + assert "gcp.bigquery.project-id" in hit[0] + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None and "binding.location.project" in refusal + + +def _warehouse_warnings(contract: Dict[str, Any]): + return [w for w in validate_iceberg_sink(contract)[1] if "no gs:// warehouse" in w] + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_bigquery_with_no_derivable_warehouse_warns(engine): + contract = _contract(_bq(project="acme-proj"), engine=engine) + errors = validate_iceberg_sink(contract)[0] + assert errors == [] + hit = _warehouse_warnings(contract) + assert len(hit) == 1 + assert "default storage location URI" in hit[0] + assert ("debezium.sink.iceberg.warehouse" in hit[0]) == (engine == "debezium") + + +def test_bigquery_with_a_bucket_does_not_warn_about_the_warehouse(): + assert _warehouse_warnings(_contract(_FULL_BQ)) == [] + + +@pytest.mark.parametrize( + "engine, props", + [ + ( + "kafka-connect", + {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "gs://ops/wh"}}, + ), + ( + "debezium", + { + "deployment": {"mode": "embedded"}, + "server": { + "sink": {"iceberg_sink_enabled": True, "config": {"warehouse": "gs://o"}} + }, + }, + ), + ], +) +def test_an_override_warehouse_silences_the_bigquery_warning(engine, props): + contract = _contract(_bq(project="acme-proj"), engine=engine, props=props) + assert _warehouse_warnings(contract) == [] + + +def test_runtime_version_warning_names_the_failure_and_the_remedy(): + warnings = validate_iceberg_sink(_contract(_FULL_BQ))[1] + hit = [w for w in warnings if "iceberg.catalog.type=bigquery" in w] + assert len(hit) == 1 + assert "on that sink the connector fails at start" in hit[0] + assert "Run a sink built from Iceberg >= 1.10" in hit[0] diff --git a/tests/build_runners/test_iceberg_sink_validation.py b/tests/build_runners/test_iceberg_sink_validation.py index 48b8b504..8d489058 100644 --- a/tests/build_runners/test_iceberg_sink_validation.py +++ b/tests/build_runners/test_iceberg_sink_validation.py @@ -695,7 +695,8 @@ def test_gcp_sink_catalog_bigquery_agrees_with_biglake_and_warns_the_runtime(): # "disagrees with 'rest'" error (dbt and the IaC never read 'rest' here). # The published sink still cannot load type=bigquery, which is a warning. contract = _sink_contract( - _loc_binding(platform="gcp", warehouse="gs://acme/wh"), sink={"catalog": "bigquery"} + _loc_binding(platform="gcp", warehouse="gs://acme/wh", project="acme-proj"), + sink={"catalog": "bigquery"}, ) errors, warnings = validate_iceberg_sink(contract) assert errors == [] @@ -735,7 +736,7 @@ def test_bigquery_warning_follows_the_type_that_reaches_the_worker(): @pytest.mark.parametrize( "catalog, location", [ - ("bigquery", {"warehouse": "gs://acme/wh"}), + ("bigquery", {"warehouse": "gs://acme/wh", "project": "acme-proj"}), ("rest", _GCP_REST_LOC), ("lakekeeper", COMPLETE), ], @@ -958,7 +959,9 @@ def test_unnamed_catalog_off_gcp_is_not_refused(): def test_bigquery_on_kafka_connect_warns_about_the_published_sink(): - contract = _sink_contract(_loc_binding("bigquery", platform="gcp", warehouse="gs://acme/wh")) + contract = _sink_contract( + _loc_binding("bigquery", platform="gcp", warehouse="gs://acme/wh", project="acme-proj") + ) errors, warnings = validate_iceberg_sink(contract) assert errors == [] hit = [w for w in warnings if "iceberg.catalog.type=bigquery" in w] diff --git a/tests/policy/test_fw2_confluent_default_catalog.py b/tests/policy/test_fw2_confluent_default_catalog.py new file mode 100644 index 00000000..3a4ec1bd --- /dev/null +++ b/tests/policy/test_fw2_confluent_default_catalog.py @@ -0,0 +1,105 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CON-5: a Confluent Tableflow expose with no catalog reads as Glue everywhere. + +The Tableflow IaC publishes such a table to AWS Glue, while the kind table +used to call it ``rest``: policy compile dropped every grant with a false +"cataloged in 'rest'" warning, and dbt-snowflake lost the Glue-linked +database type. Absent and ``catalog: glue`` must now compile the same. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, Optional + +import pytest +import yaml + +from fluid_build.engines.dbt.catalogs_yml import generate_catalogs_yml +from fluid_build.iac import build_module, get_iac_plugin +from fluid_build.iac.providers.confluent import validate_confluent_binding +from fluid_build.policy.compiler import compile_policy + +pytestmark = [pytest.mark.unit] + + +def _contract(catalog: Optional[str]) -> Dict[str, Any]: + loc: Dict[str, Any] = { + "environment_id": "env-123", + "kafka_cluster_id": "lkc-123", + "topic": "orders", + "database": "sales_glue", + "bucket": "acme-tableflow", + "confluent_role_arn": "arn:aws:iam::123456789012:role/tableflow", + "region": "eu-west-1", + } + if catalog is not None: + loc["catalog"] = catalog + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.tableflow", + "name": "Tableflow orders", + "metadata": {"layer": "Bronze"}, + "accessPolicy": { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + }, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": {"platform": "confluent", "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "order_id", "type": "integer"}]}, + } + ], + } + + +def test_policy_compile_grants_the_glue_table_when_catalog_is_absent(): + bindings, warnings = compile_policy(_contract(None)) + assert not any("cataloged in" in w for w in warnings), warnings + assert [(b["provider"], b["resource_type"], b["resource_id"]) for b in bindings] == [ + ("aws", "s3.bucket", "acme-tableflow"), + ("aws", "glue.table", "sales_glue"), + ] + + +def test_policy_compile_is_the_same_for_absent_and_explicit_glue(): + assert compile_policy(_contract(None)) == compile_policy(_contract("glue")) + + +def test_dbt_snowflake_links_the_glue_database_when_catalog_is_absent(): + build = {"engine": "dbt", "execution": {"runtime": {"platform": "snowflake"}}} + absent = generate_catalogs_yml(_contract(None), build) + assert absent == generate_catalogs_yml(_contract("glue"), build) + integration = yaml.safe_load(absent)["catalogs"][0]["write_integrations"][0] + assert integration["adapter_properties"]["catalog_linked_database_type"] == "glue" + + +def test_tableflow_iac_still_publishes_an_absent_catalog_to_glue(): + plugin = get_iac_plugin("confluent") + absent = json.loads(build_module(plugin, _contract(None)))["resource"] + glue = next(iter(absent["confluent_catalog_integration"].values()))["aws_glue"] + assert glue["custom_database"] == "sales_glue" + assert build_module(plugin, _contract(None)) == build_module(plugin, _contract("glue")) + assert validate_confluent_binding(_contract(None)) == validate_confluent_binding( + _contract("glue") + ) + + +def test_a_non_glue_confluent_catalog_is_still_refused(): + errors, _ = validate_confluent_binding(_contract("lakekeeper")) + assert errors and "lakekeeper" in errors[0] diff --git a/tests/providers/test_fw2_catalog_table.py b/tests/providers/test_fw2_catalog_table.py new file mode 100644 index 00000000..edd26adc --- /dev/null +++ b/tests/providers/test_fw2_catalog_table.py @@ -0,0 +1,184 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The catalog-kind table against what each Iceberg catalog needs to start. + +CON-3: Iceberg's DynamoDbCatalog and JdbcCatalog refuse to initialise without +a warehouse (apache-iceberg-1.10.0 DynamoDbCatalog.java:132-134, +JdbcCatalog.java:117-119), so the resolver derives one from an explicit +``location.bucket``. CON-4: BigQueryMetastoreCatalog refuses to initialise +without ``gcp.bigquery.project-id`` (BigQueryMetastoreCatalog.java:84-87). +CON-5: a ``platform: confluent`` expose with no catalog is the Glue table the +Tableflow IaC publishes. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.providers._iceberg_catalog import ( + BIGQUERY_LOCATION, + BIGQUERY_PROJECT_ID, + binding_catalog_kind, + catalog_kind_info, + default_catalog_kind, + is_glue_cataloged, + resolve_iceberg_catalog, +) + +pytestmark = [pytest.mark.unit] + + +def _binding(catalog: Optional[str], platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = {"database": "streaming", "table": "orders", **location} + if catalog is not None: + loc["catalog"] = catalog + return {"platform": platform, "format": "iceberg", "location": loc} + + +# ── the rows: an independent pin of what each sink needs ──────────────────── + + +@pytest.mark.parametrize( + "kind, requires", + [ + ("dynamodb", ("warehouse",)), + ("jdbc", ("uri", "warehouse")), + ("bigquery", ("project",)), + ], +) +def test_sink_requires_matches_the_catalog_initialize_checks(kind, requires): + assert catalog_kind_info(kind).sink_requires == requires + + +# ── CON-3: dynamodb / jdbc derive a warehouse from an explicit bucket ─────── + + +@pytest.mark.parametrize( + "kind, extra", [("dynamodb", {}), ("jdbc", {"uri": "jdbc:postgresql://pg:5432/iceberg"})] +) +@pytest.mark.parametrize( + "platform, expected", + [ + ("aws", "s3://acme-lake/streaming/orders/"), + ("s3", "s3://acme-lake/streaming/orders/"), # an aws alias + ("gcp", "gs://acme-lake/streaming/orders/"), + ], +) +def test_bucket_derives_the_warehouse_with_the_platform_scheme(kind, extra, platform, expected): + r = resolve_iceberg_catalog(_binding(kind, platform, bucket="acme-lake", **extra)) + assert r.warehouse == expected + + +def test_derived_warehouse_honours_location_path(): + r = resolve_iceberg_catalog(_binding("dynamodb", bucket="acme-lake", path="/lake/orders")) + assert r.warehouse == "s3://acme-lake/lake/orders" + + +def test_explicit_warehouse_beats_the_bucket(): + r = resolve_iceberg_catalog(_binding("dynamodb", bucket="acme-lake", warehouse="s3://other/wh")) + assert r.warehouse == "s3://other/wh" + + +@pytest.mark.parametrize( + "binding", + [ + # no bucket: never the account-derived bucket the Glue row falls back to + _binding("dynamodb"), + _binding("jdbc", uri="jdbc:postgresql://pg:5432/iceberg"), + # an unresolved template is not a bucket either + _binding("dynamodb", bucket="{{ env.FW2_NO_SUCH_BUCKET_VAR }}"), + # a platform with no scheme to derive with + _binding("jdbc", "local", bucket="acme-lake", uri="jdbc:sqlite::memory:"), + ], +) +def test_no_explicit_bucket_derives_no_warehouse(binding, monkeypatch): + monkeypatch.delenv("FW2_NO_SUCH_BUCKET_VAR", raising=False) + assert resolve_iceberg_catalog(binding, account_ref="123456789012").warehouse == "" + + +def test_glue_keeps_its_account_fallback(): + r = resolve_iceberg_catalog(_binding("glue"), account_ref="123456789012") + assert r.warehouse == "s3://123456789012-fluid-data/streaming/orders/" + + +@pytest.mark.parametrize("kind", ["hadoop", "hive", "nessie", "rest", "lakekeeper"]) +def test_other_kinds_do_not_derive_from_the_bucket(kind): + assert resolve_iceberg_catalog(_binding(kind, bucket="acme-lake")).warehouse == "" + + +# ── CON-4: bigquery gets its project, location and gs:// warehouse ────────── + + +def _bq(**location: Any) -> Dict[str, Any]: + return _binding("bigquery", "gcp", **location) + + +def test_bigquery_maps_project_and_region_to_its_own_properties(): + r = resolve_iceberg_catalog(_bq(project="acme-proj", region="eu", bucket="acme-lake")) + assert dict(r.extra_catalog_props) == { + BIGQUERY_PROJECT_ID: "acme-proj", + BIGQUERY_LOCATION: "eu", + } + assert BIGQUERY_PROJECT_ID == "gcp.bigquery.project-id" + assert BIGQUERY_LOCATION == "gcp.bigquery.location" + # client.region is an AWS client property + assert r.region is None + + +def test_bigquery_without_region_sets_no_location(): + r = resolve_iceberg_catalog(_bq(project="acme-proj")) + assert dict(r.extra_catalog_props) == {BIGQUERY_PROJECT_ID: "acme-proj"} + + +@pytest.mark.parametrize( + "location, warehouse", + [ + ({"bucket": "acme-lake"}, "gs://acme-lake"), + ({"bucket": "acme-lake", "path": "iceberg"}, "gs://acme-lake/iceberg"), + ({"warehouse": "gs://wh-bucket/root", "bucket": "acme-lake"}, "gs://wh-bucket/root"), + # a foreign scheme is storage the GCP IaC never creates + ({"warehouse": "s3://elsewhere"}, ""), + ({}, ""), + ], +) +def test_bigquery_warehouse_is_the_gs_storage_uri(location, warehouse): + r = resolve_iceberg_catalog(_bq(project="acme-proj", **location)) + assert r.warehouse == warehouse + assert (r.io_impl is not None) == bool(warehouse) + + +# ── CON-5: confluent's default catalog is Glue, as Tableflow publishes ────── + + +def test_confluent_default_catalog_is_glue(): + binding = {"platform": "confluent", "format": "iceberg", "location": {"database": "d"}} + assert default_catalog_kind(binding) == "glue" + assert binding_catalog_kind(binding) == "glue" + assert is_glue_cataloged(binding) + + +@pytest.mark.parametrize( + "platform, expected", + [("aws", "glue"), ("snowflake", "snowflake-managed"), ("gcp", "rest"), ("local", "rest")], +) +def test_other_platform_defaults_are_unchanged(platform, expected): + assert default_catalog_kind({"platform": platform}) == expected + + +def test_confluent_explicit_catalog_still_wins(): + binding = {"platform": "confluent", "location": {"catalog": "lakekeeper"}} + assert binding_catalog_kind(binding) == "lakekeeper" From c519e18b4ace9bb422d1f0dcaadac7a484584251 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:14:44 +0200 Subject: [PATCH 05/19] fix(iceberg): an override can supply the warehouse or project a catalog needs to start The DynamoDB / JDBC warehouse and the BigQuery gcp.bigquery.project-id requirements read only binding.location, so validate and the preflight refused a config whose pushed form carried the key: a Kafka Connect iceberg_catalog_overrides or sink_connector_config entry, or a Debezium server.sink.config entry. Main accepted these. Check 4 now counts the key when an override map sets the catalog property under the runtime's prefix (iceberg.catalog. on Kafka Connect, bare on Debezium Server), and still errors when neither the binding nor an override supplies it. The refusal names the override remedy. The BigQuery no-warehouse warning understated the Kafka Connect case: its auto-create calls createNamespace before every createTable, and BigQueryMetastoreCatalog.createNamespace refuses without a warehouse, so table auto-creation fails even in a dataset that exists (apache-iceberg-1.10.0 IcebergWriterFactory.java:89, :122-137; BigQueryMetastoreCatalog.java:197-208, :298-303). The Debezium Server sink creates a namespace only when it is missing, so its wording keeps the dataset storage URI exception (debezium-server-iceberg 1.2.0.Final IcebergUtil.java:98-126). --- .../kafka_connect/iceberg_sink_validation.py | 76 ++++-- .../test_fw2_catalog_table_overrides.py | 233 ++++++++++++++++++ .../test_fw2_catalog_table_sink.py | 1 - 3 files changed, 291 insertions(+), 19 deletions(-) create mode 100644 tests/build_runners/test_fw2_catalog_table_overrides.py diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 82bf5a68..f9bb8a99 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -388,11 +388,17 @@ def _check_catalog( # 4. catalog tagged-union completeness, from the row's ``sink_requires``. # HARD: forge can't derive a REST uri or a catalog name. A warehouse is # present when the deriver resolves one (DynamoDB and JDBC derive it - # from an explicit bucket). Glue requires nothing (the warehouse falls + # from an explicit bucket). A key the catalog needs only to start (the + # DynamoDB / JDBC warehouse, the BigQuery project) is also present when + # an override map sets its catalog property: the runner merges that map + # into the config it pushes. Glue requires nothing (the warehouse falls # back), so its region stays advisory. derived_warehouse = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse for key in info.sink_requires: present = derived_warehouse if key == "warehouse" else loc.get(key) + start_prop = _start_only_property(key, kind) + if not present and start_prop is not None: + present = _overrides_catalog_property(runtime, start_prop) if not present: errors.append( f"iceberg sink (build {bid!r}): {kind} catalog requires " @@ -428,11 +434,17 @@ def _check_catalog( "Iceberg's CatalogUtil gains the bigquery type in 1.10, so on that sink the " "connector fails at start. Run a sink built from Iceberg >= 1.10" ) - # BigQueryMetastoreCatalog reads the warehouse only to place a dataset it - # creates, or a table in a dataset with no default storage location URI - # (apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:147-160, :197-205, - # :298-303); an auto-creating sink creates both (IcebergWriterFactory.java - # :89, :122-131). Advisory: a dataset with a storage URI needs none. + # BigQueryMetastoreCatalog reads the warehouse to place a dataset it + # creates, and a table it creates in a dataset with no default storage + # location URI; without one, both throw IllegalArgumentException + # (apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:146-161, :197-208, + # :298-303). The Kafka Connect sink's auto-create calls createNamespace + # before every createTable and catches only AlreadyExists / Forbidden + # (IcebergWriterFactory.java:89, :122-137), so every table auto-create + # throws, even in a dataset that exists. The Debezium Server sink creates + # a missing table, and its dataset only when the dataset is missing + # (memiiso/debezium-server-iceberg 1.2.0.Final IcebergChangeConsumer.java + # :322-325, IcebergUtil.java:98-126). Advisory: tables that exist need none. if ( reaching == "bigquery" and kind == "bigquery" @@ -443,13 +455,18 @@ def _check_catalog( warnings.append( f"iceberg sink (build {bid!r}): no gs:// warehouse can be derived from " "binding.location (a gs:// warehouse, or a bucket), so the derived bigquery " - "catalog config sets none. BigQueryMetastoreCatalog needs one to create a " - "dataset, or a table in a dataset with no default storage location URI" + "catalog config sets none. " + ( - "; the Debezium Server Iceberg sink also declares " + "The Debezium Server Iceberg sink creates a missing table, which " + "BigQueryMetastoreCatalog refuses without a warehouse unless the table's " + "dataset exists with a default storage location URI (a missing dataset it " + "refuses outright); tables that exist need none. The sink also declares " "debezium.sink.iceberg.warehouse with no default" if runtime.engine == "debezium" - else "" + else "With iceberg.tables.auto-create-enabled the sink calls createNamespace " + "for every table it creates, and BigQueryMetastoreCatalog.createNamespace " + "refuses without a warehouse, so table auto-creation fails even in a dataset " + "that exists; without auto-create, tables that exist need none" ) ) @@ -464,25 +481,46 @@ def _requires_hint(key: str, kind: str, info: CatalogKind) -> str: if key == "warehouse" and kind in BUCKET_WAREHOUSE_KINDS: return ( " (an object-store location), or a binding.location.bucket on platform aws or " - f"gcp to derive it from; the {kind} catalog refuses to start without a warehouse" + "gcp to derive it from, or the sink's warehouse property in an override; the " + f"{kind} catalog refuses to start without a warehouse" ) if key == "project" and kind == "bigquery": return ( - f" (the sink's {BIGQUERY_PROJECT_ID}); the bigquery catalog refuses to start " - "without it" + f" (the sink's {BIGQUERY_PROJECT_ID}), or that property in an override; the " + "bigquery catalog refuses to start without it" ) return "" -def _overrides_warehouse(runtime: IcebergSinkPlan) -> bool: - """Does an override map merged over the derived config set the warehouse?""" - # The warehouse key carries the same runtime prefix as the type key: +def _start_only_property(key: str, kind: str) -> Optional[str]: + """The catalog property behind a ``sink_requires`` key only the sink needs. + + DynamoDbCatalog and JdbcCatalog refuse to start without ``warehouse``, and + BigQueryMetastoreCatalog without ``gcp.bigquery.project-id`` (the rows in + ``_iceberg_catalog`` cite the upstream lines), so an override map that sets + the property supplies what the sink needs. ``None`` for any other key. + """ + if key == "warehouse" and kind in BUCKET_WAREHOUSE_KINDS: + return "warehouse" + if key == "project" and kind == "bigquery": + return BIGQUERY_PROJECT_ID + return None + + +def _overrides_catalog_property(runtime: IcebergSinkPlan, prop: str) -> bool: + """Does an override map merged over the derived config set catalog property ``prop``?""" + # A catalog property carries the same runtime prefix as the type key: # ``iceberg.catalog.warehouse`` on Kafka Connect, bare ``warehouse`` on # Debezium Server. - key = runtime.type_key[: -len("type")] + "warehouse" + key = runtime.type_key[: -len("type")] + prop return any(mapping.get(key) for _label, mapping in runtime.overrides) +def _overrides_warehouse(runtime: IcebergSinkPlan) -> bool: + """Does an override map merged over the derived config set the warehouse?""" + return _overrides_catalog_property(runtime, "warehouse") + + def _reaching_catalog_type(info: Any, runtime: IcebergSinkPlan) -> Optional[str]: """The catalog ``type`` the worker actually receives, or ``None``. @@ -629,7 +667,9 @@ def _check_warehouse_override( derived_wh = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse if not derived_wh and "warehouse" in info.sink_requires: - return # check 4 already refused the missing warehouse + # The binding has no warehouse to diverge from: check 4 refused it, or + # (DynamoDB / JDBC) accepted the override as the sink's warehouse. + return if same_warehouse(override_wh, derived_wh): return if info.family == FAMILY_GLUE: diff --git a/tests/build_runners/test_fw2_catalog_table_overrides.py b/tests/build_runners/test_fw2_catalog_table_overrides.py new file mode 100644 index 00000000..b125cbf5 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_overrides.py @@ -0,0 +1,233 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An override or a hand-written config can supply what a catalog needs to start. + +The DynamoDB / JDBC warehouse and the BigQuery ``gcp.bigquery.project-id`` are +hard requirements of the sink, but the runner merges operator maps into the +config it pushes, so a key set there reaches the worker. Validate and the +preflight used to read only ``binding.location`` for them and refused configs +whose pushed form carried the key. The prefix is the runtime's: Kafka Connect +reads ``iceberg.catalog.``, Debezium Server takes the bare ````. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import pytest + +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import DYNAMODB_CATALOG_IMPL, resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + +_JDBC_URI = "jdbc:postgresql://pg:5432/iceberg" + + +def _binding(catalog: str, platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc = {"catalog": catalog, "database": "streaming", "table": "orders", **location} + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _kc_contract(binding: Dict[str, Any], kc: Dict[str, Any]) -> Dict[str, Any]: + return _contract(binding, "kafka-connect", kc) + + +def _dbz_contract(binding: Dict[str, Any], server_sink: Dict[str, Any]) -> Dict[str, Any]: + return _contract( + binding, "debezium", {"deployment": {"mode": "embedded"}, "server": {"sink": server_sink}} + ) + + +def _contract(binding: Dict[str, Any], engine: str, engine_props: Dict[str, Any]): + return { + "id": "bronze.orders_stream", + "builds": [ + { + "id": "ingest", + "engine": engine, + "properties": { + "source": {"kind": "postgres", "mode": "incremental_append"}, + "sink": {"format": "iceberg"}, + engine: engine_props, + }, + } + ], + "exposes": [{"exposeId": "orders", "kind": "table", "binding": binding}], + } + + +def _accepted(contract: Dict[str, Any]) -> None: + errors = validate_iceberg_sink(contract)[0] + assert errors == [], errors + assert iceberg_sink_preflight(contract, "ingest") is None + + +def _refused(contract: Dict[str, Any], needle: str) -> None: + errors = validate_iceberg_sink(contract)[0] + assert any(needle in e for e in errors), errors + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None and needle in refusal + + +_NO_WH = { + "dynamodb": _binding("dynamodb", region="eu-west-1"), + "jdbc": _binding("jdbc", region="eu-west-1", uri=_JDBC_URI), +} + + +# ── warehouse: DynamoDB / JDBC ────────────────────────────────────────────── + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_kafka_connect_catalog_override_warehouse_is_accepted(kind): + kc = {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "s3://ops/wh"}} + _accepted(_kc_contract(_NO_WH[kind], kc)) + # ...and it is what the runner pushes. + resolved = resolve_iceberg_catalog(_NO_WH[kind], account_ref="") + cfg = emit_iceberg_sink_config(resolved, product_id="p", topics=["t"], kc_props=kc) + assert cfg["iceberg.catalog.warehouse"] == "s3://ops/wh" + + +def test_kafka_connect_hand_written_config_with_a_warehouse_is_accepted(): + kc = { + "sink_connector_config": { + "connector.class": "x", + "iceberg.catalog.catalog-impl": DYNAMODB_CATALOG_IMPL, + "iceberg.catalog.warehouse": "s3://ops/wh", + } + } + _accepted(_kc_contract(_NO_WH["dynamodb"], kc)) + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("enabled", [True, False], ids=["derived", "hand-written"]) +def test_debezium_server_sink_config_warehouse_is_accepted(kind, enabled): + server_sink = {"iceberg_sink_enabled": enabled, "config": {"warehouse": "s3://ops/wh"}} + if not enabled and kind == "dynamodb": + server_sink["config"]["catalog-impl"] = DYNAMODB_CATALOG_IMPL + if not enabled and kind == "jdbc": + server_sink["config"]["type"] = "jdbc" + _accepted(_dbz_contract(_NO_WH[kind], server_sink)) + + +@pytest.mark.parametrize( + "kc", + [ + {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": ""}}, + # Kafka Connect does not read a bare ``warehouse`` as a catalog property + {"iceberg_catalog_overrides": {"warehouse": "s3://ops/wh"}}, + {"iceberg_catalog_overrides": {"iceberg.catalog.client.region": "eu-west-1"}}, + ], + ids=["empty", "unprefixed", "other-key"], +) +def test_an_override_that_sets_no_catalog_warehouse_is_still_refused(kc): + _refused( + _kc_contract(_NO_WH["dynamodb"], kc), "dynamodb catalog requires binding.location.warehouse" + ) + + +def test_debezium_prefixed_warehouse_does_not_count(): + server_sink = {"iceberg_sink_enabled": True, "config": {"iceberg.catalog.warehouse": "s3://o"}} + _refused( + _dbz_contract(_NO_WH["dynamodb"], server_sink), + "dynamodb catalog requires binding.location.warehouse", + ) + + +def test_an_override_warehouse_does_not_satisfy_the_jdbc_uri(): + binding = _binding("jdbc", region="eu-west-1") + kc = {"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "s3://ops/wh"}} + errors = validate_iceberg_sink(_kc_contract(binding, kc))[0] + assert [e for e in errors if "requires binding.location" in e] == [ + "iceberg sink (build 'ingest'): jdbc catalog requires binding.location.uri" + ] + + +def test_the_refusal_names_the_override_remedy(): + errors = validate_iceberg_sink(_kc_contract(_NO_WH["dynamodb"], {}))[0] + hit = [e for e in errors if "requires binding.location.warehouse" in e] + assert len(hit) == 1 and "in an override" in hit[0] + + +# ── project: BigQuery ─────────────────────────────────────────────────────── + +_BQ_NO_PROJECT = _binding("bigquery", "gcp", bucket="lake") + + +def test_kafka_connect_catalog_override_project_is_accepted(): + kc = {"iceberg_catalog_overrides": {"iceberg.catalog.gcp.bigquery.project-id": "p"}} + _accepted(_kc_contract(_BQ_NO_PROJECT, kc)) + + +def test_kafka_connect_hand_written_project_is_accepted(): + kc = { + "sink_connector_config": { + "iceberg.catalog.type": "bigquery", + "iceberg.catalog.gcp.bigquery.project-id": "p", + } + } + _accepted(_kc_contract(_BQ_NO_PROJECT, kc)) + + +def test_debezium_server_sink_config_project_is_accepted(): + server_sink = {"iceberg_sink_enabled": True, "config": {"gcp.bigquery.project-id": "p"}} + _accepted(_dbz_contract(_BQ_NO_PROJECT, server_sink)) + + +@pytest.mark.parametrize( + "overrides", + [{"iceberg.catalog.gcp.bigquery.project-id": ""}, {"gcp.bigquery.project-id": "p"}], + ids=["empty", "unprefixed"], +) +def test_an_override_that_sets_no_project_is_still_refused(overrides): + _refused( + _kc_contract(_BQ_NO_PROJECT, {"iceberg_catalog_overrides": overrides}), + "bigquery catalog requires binding.location.project", + ) + + +# ── the BigQuery no-warehouse warning ─────────────────────────────────────── + + +def _warehouse_warning(contract: Dict[str, Any]) -> str: + hit = [w for w in validate_iceberg_sink(contract)[1] if "no gs:// warehouse" in w] + assert len(hit) == 1, hit + return hit[0] + + +def test_kafka_connect_warning_says_auto_create_fails_even_in_an_existing_dataset(): + # apache-iceberg-1.10.0: IcebergWriterFactory.autoCreateTable calls + # createNamespace before createTable, and BigQueryMetastoreCatalog's + # createNamespace refuses without a warehouse. + binding = _binding("bigquery", "gcp", project="acme-proj", region="EU") + warning = _warehouse_warning(_kc_contract(binding, {"streamingSink": {"autoCreate": True}})) + assert "createNamespace" in warning + assert "table auto-creation fails even in a dataset that exists" in warning + assert "default storage location URI" not in warning + + +def test_debezium_warning_keeps_the_dataset_storage_uri_exception(): + # memiiso/debezium-server-iceberg 1.2.0.Final creates a namespace only when + # it is missing, so an existing dataset's storage URI places a new table. + binding = _binding("bigquery", "gcp", project="acme-proj") + warning = _warehouse_warning(_dbz_contract(binding, {})) + assert "default storage location URI" in warning + assert "debezium.sink.iceberg.warehouse" in warning + assert "createNamespace" not in warning diff --git a/tests/build_runners/test_fw2_catalog_table_sink.py b/tests/build_runners/test_fw2_catalog_table_sink.py index 029a62a3..2d35f9cf 100644 --- a/tests/build_runners/test_fw2_catalog_table_sink.py +++ b/tests/build_runners/test_fw2_catalog_table_sink.py @@ -193,7 +193,6 @@ def test_bigquery_with_no_derivable_warehouse_warns(engine): assert errors == [] hit = _warehouse_warnings(contract) assert len(hit) == 1 - assert "default storage location URI" in hit[0] assert ("debezium.sink.iceberg.warehouse" in hit[0]) == (engine == "debezium") From cc8338eb8a835e99481aff2ae0cbda52f229a215 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:02:46 +0200 Subject: [PATCH 06/19] fix(iceberg): an env-templated bucket warns at validate and is refused only by the runner preflight A dynamodb or jdbc expose whose location.bucket is an unresolved {{ env.* }} template was treated as having no bucket: fluid validate failed in a shell without the variable and asked for the bucket the contract already sets. Validate now warns and names the unset variables; the runner preflight, in the environment the sink config is derived in, refuses the build. --- .../kafka_connect/iceberg_sink_validation.py | 62 ++++- fluid_build/providers/_iceberg_catalog.py | 31 ++- .../test_fw2_catalog_table_envbucket.py | 261 ++++++++++++++++++ 3 files changed, 347 insertions(+), 7 deletions(-) create mode 100644 tests/build_runners/test_fw2_catalog_table_envbucket.py diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index f9bb8a99..a6f7d492 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -45,8 +45,9 @@ hand-written config selects (its ``type`` or ``catalog-impl``); * every ``binding.location`` key in the row's ``sink_requires`` must be set, a warehouse counting when the deriver resolves one (DynamoDB and JDBC derive - it from an explicit bucket); Glue keeps its advisory region warning: the - warehouse falls back; + it from an explicit bucket; a bucket ``{{ env.* }}`` template that does not + resolve is a warning here and an error in the runner's preflight); Glue + keeps its advisory region warning: the warehouse falls back; * the runtime must ship the catalog's client (the stock Apache Iceberg Kafka Connect runtime has no Nessie client, and the published sink predates the ``bigquery`` catalog type); @@ -72,6 +73,7 @@ from __future__ import annotations +import contextvars import logging from dataclasses import dataclass from typing import AbstractSet, Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -244,12 +246,19 @@ def iceberg_sink_preflight( a different catalog than dbt reads. ``build_id`` is the run context's id; the build is :func:`executing_build`, the one the runner reads its properties from, not one matched by message text. Warnings are logged and - never fatal, the same split as ``fluid validate`` without ``--strict``. + never fatal, the same split as ``fluid validate`` without ``--strict``, with + one difference: a ``location.bucket`` template that ``fluid validate`` + only warns about (it may resolve where the sink runs) is an error here, + because the runner derives the sink config in this process next. """ build = executing_build(contract, build_id) if build is None: return None - errors, warnings = _validate(contract, builds=(build,)) + token = _AT_RUN.set(True) + try: + errors, warnings = _validate(contract, builds=(build,)) + finally: + _AT_RUN.reset(token) for msg in warnings: (log or LOG).warning("iceberg_sink.preflight.warning build=%s %s", build_id, msg) if not errors: @@ -391,15 +400,24 @@ def _check_catalog( # from an explicit bucket). A key the catalog needs only to start (the # DynamoDB / JDBC warehouse, the BigQuery project) is also present when # an override map sets its catalog property: the runner merges that map - # into the config it pushes. Glue requires nothing (the warehouse falls + # into the config it pushes. A bucket whose ``{{ env.* }}`` template + # does not resolve here is explicit, so ``fluid validate`` only warns: + # the runner's preflight, in the environment the sink config is + # derived in, refuses it. Glue requires nothing (the warehouse falls # back), so its region stays advisory. + from ...providers._iceberg_catalog import unset_bucket_env_vars + derived_warehouse = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse for key in info.sink_requires: present = derived_warehouse if key == "warehouse" else loc.get(key) start_prop = _start_only_property(key, kind) if not present and start_prop is not None: present = _overrides_catalog_property(runtime, start_prop) - if not present: + unset = unset_bucket_env_vars(binding) if key == "warehouse" else () + if not present and unset and kind in BUCKET_WAREHOUSE_KINDS: + issue = _unresolved_bucket_issue(bid, kind, binding, unset) + (errors if _AT_RUN.get() else warnings).append(issue) + elif not present: errors.append( f"iceberg sink (build {bid!r}): {kind} catalog requires " f"binding.location.{key}{_requires_hint(key, kind, info)}" @@ -474,6 +492,38 @@ def _check_catalog( _check_selector_overrides(bid, kind, info, runtime, errors) +#: True while :func:`iceberg_sink_preflight` runs. The runner derives the sink +#: config in the same process right after it, so a ``location.bucket`` +#: template this environment cannot resolve leaves the sink with no warehouse. +_AT_RUN: contextvars.ContextVar[bool] = contextvars.ContextVar( + "iceberg_sink_preflight", default=False +) + + +def _unresolved_bucket_issue( + bid: Any, kind: str, binding: Mapping[str, Any], names: Tuple[str, ...] +) -> str: + """Check 4's message for a warehouse that waits on unset bucket variables.""" + raw = (binding.get("location") or {}).get("bucket") + one = len(names) == 1 + unset = f"{', '.join(names)} {'is' if one else 'are'} not set" + if _AT_RUN.get(): + return ( + f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " + f"binding.location.bucket {raw!r}, and {unset} in the runner's environment, so " + f"the sink would get no warehouse and the {kind} catalog refuses to start " + f"without one. Set {'it' if one else 'them'} here, or set " + "binding.location.warehouse" + ) + return ( + f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " + f"binding.location.bucket {raw!r}, and {unset} here, so it cannot be derived at " + f"validate time. Set {'it' if one else 'them'} where the sink runs: the runner " + "refuses the build when the bucket does not resolve there, because the " + f"{kind} catalog refuses to start without a warehouse" + ) + + def _requires_hint(key: str, kind: str, info: CatalogKind) -> str: """What a missing ``sink_requires`` key is to the sink, for check 4's error.""" if key == "warehouse" and info.family == FAMILY_REST: diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index b8a72f9c..7c0ca345 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -32,7 +32,13 @@ from typing import Any, Dict, Mapping, Optional, Tuple from ._sql_safety import validate_ident -from .aws.util.warehouse import bucket_uses_fallback, get_iceberg_warehouse, normalize_location +from .aws.util.warehouse import ( + _ENV_TEMPLATE_RE, + bucket_uses_fallback, + get_iceberg_warehouse, + normalize_location, + resolve_env_templates, +) # Apache Iceberg runtime class names (pinned to the connector surface validated # in the OSS spike — RFC §14). Bumping the Iceberg runtime may change these. @@ -495,6 +501,29 @@ def _bucket_warehouse(binding: Mapping[str, Any]) -> str: return f"{scheme}://{bucket}/{path}" +def unset_bucket_env_vars(binding: Mapping[str, Any]) -> Tuple[str, ...]: + """The unset variables an explicit bucket's derived warehouse waits on, or ``()``. + + Non-empty when ``location.bucket`` is set on a platform + :func:`_bucket_warehouse` derives on (aws, gcp) but its ``{{ env.* }}`` + templates do not resolve in this process's environment: the bucket is + explicit, so the warehouse derives wherever those variables are set, but + not here. ``()`` for a resolved or absent bucket, another platform, or a + template that is not ``{{ env.* }}`` (nothing resolves that one). + """ + from ..iac.provider_match import canonical_cloud + + loc = binding.get("location") or {} + raw = loc.get("bucket") + if not raw or canonical_cloud(binding.get("platform")) not in _BUCKET_WAREHOUSE_SCHEMES: + return () + resolved = str(resolve_env_templates(raw)) + names = tuple(dict.fromkeys(_ENV_TEMPLATE_RE.findall(resolved))) + if not names or "{{" in _ENV_TEMPLATE_RE.sub("", resolved): + return () + return names + + # --------------------------------------------------------------------------- # Object-store warehouse URI # --------------------------------------------------------------------------- diff --git a/tests/build_runners/test_fw2_catalog_table_envbucket.py b/tests/build_runners/test_fw2_catalog_table_envbucket.py new file mode 100644 index 00000000..0e695217 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_envbucket.py @@ -0,0 +1,261 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An env-templated bucket is an explicit bucket, set or not where validate runs. + +DynamoDB and JDBC derive their warehouse from ``location.bucket``. A bucket +written as ``{{ env.DATA_LAKE_BUCKET }}`` (the form +``examples/aws-glue-data-lake/contract-iceberg.fluid.yaml`` uses) derives +nothing in a shell without that variable, and check 4 used to treat it as +absent: ``fluid validate`` in such a shell (a CI job, say) refused the +contract and asked for the bucket it already sets, while the same command with +the variable set accepted it. Validate now warns and names the variable; the +runner's preflight, in the environment the sink config is derived in, still +refuses when it does not resolve there. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog, unset_bucket_env_vars + +pytestmark = [pytest.mark.unit] + +_VAR = "FW2_DATA_LAKE_BUCKET" +_TEMPLATE = "{{ env.%s }}" % _VAR +_JDBC_URI = "jdbc:postgresql://pg:5432/iceberg" + + +def _binding(catalog: str, platform: str = "aws", **location: Any) -> Dict[str, Any]: + loc = {"catalog": catalog, "database": "streaming", "table": "orders", **location} + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _contract( + binding: Dict[str, Any], *, engine: str = "kafka-connect", props: Optional[Dict] = None +) -> Dict[str, Any]: + build_props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append"}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + build_props["debezium"] = props or { + "deployment": {"mode": "embedded"}, + "server": {"sink": {}}, + } + else: + build_props["kafka-connect"] = props or {} + return { + "id": "bronze.orders_stream", + "builds": [{"id": "ingest", "engine": engine, "properties": build_props}], + "exposes": [{"exposeId": "orders", "kind": "table", "binding": binding}], + } + + +_TEMPLATED = [ + _binding("dynamodb", bucket=_TEMPLATE, region="eu-west-1"), + _binding("jdbc", bucket=_TEMPLATE, region="eu-west-1", uri=_JDBC_URI), +] +_IDS = ["dynamodb", "jdbc"] + + +@pytest.fixture +def unset(monkeypatch): + monkeypatch.delenv(_VAR, raising=False) + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_unresolved_bucket_template_warns_at_validate(unset, binding, engine): + kind = binding["location"]["catalog"] + errors, warnings = validate_iceberg_sink(_contract(binding, engine=engine)) + assert errors == [] + hit = [w for w in warnings if _VAR in w] + assert len(hit) == 1, warnings + assert f"the {kind} catalog's warehouse derives from binding.location.bucket" in hit[0] + assert repr(_TEMPLATE) in hit[0] + assert f"{_VAR} is not set here" in hit[0] + # It does not ask for the bucket the contract already sets. + assert "requires binding.location.warehouse" not in hit[0] + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_unresolved_bucket_template_is_refused_by_the_preflight(unset, binding, engine): + refusal = iceberg_sink_preflight(_contract(binding, engine=engine), "ingest") + assert refusal is not None + assert f"{_VAR} is not set in the runner's environment" in refusal + assert "binding.location.warehouse" in refusal + + +def test_the_preflight_leaves_validate_in_validate_mode(unset): + contract = _contract(_TEMPLATED[0]) + assert iceberg_sink_preflight(contract, "ingest") is not None + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] and any(_VAR in w for w in warnings) + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_resolved_bucket_template_validates_clean_and_derives(monkeypatch, binding, engine): + monkeypatch.setenv(_VAR, "acme-lake") + contract = _contract(binding, engine=engine) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert not [w for w in warnings if _VAR in w or "warehouse" in w], warnings + assert iceberg_sink_preflight(contract, "ingest") is None + resolved = resolve_iceberg_catalog(binding, account_ref="123456789012") + if engine == "debezium": + warehouse = emit_debezium_iceberg_sink_config(resolved)["warehouse"] + else: + cfg = emit_iceberg_sink_config(resolved, product_id="bronze.orders_stream", topics=["t"]) + warehouse = cfg["iceberg.catalog.warehouse"] + assert warehouse == "s3://acme-lake/streaming/orders/" + + +def test_resolved_bucket_template_on_gcp_derives_gs(monkeypatch): + monkeypatch.setenv(_VAR, "acme-lake") + binding = _binding("jdbc", "gcp", bucket=_TEMPLATE, uri=_JDBC_URI) + assert resolve_iceberg_catalog(binding).warehouse == "gs://acme-lake/streaming/orders/" + + +def test_several_unset_variables_are_all_named(monkeypatch): + monkeypatch.delenv("FW2_A", raising=False) + monkeypatch.delenv("FW2_B", raising=False) + binding = _binding("dynamodb", bucket="{{ env.FW2_A }}-{{ env.FW2_B }}", region="eu-west-1") + assert unset_bucket_env_vars(binding) == ("FW2_A", "FW2_B") + warnings = validate_iceberg_sink(_contract(binding))[1] + assert any("FW2_A, FW2_B are not set here" in w and "Set them" in w for w in warnings) + + +@pytest.mark.parametrize( + "binding", + [ + # nothing resolves a template that is not {{ env.* }} + _binding("dynamodb", bucket="{{ var.lake }}", region="eu-west-1"), + _binding("dynamodb", bucket="{{ env.%s }}-{{ var.x }}" % _VAR, region="eu-west-1"), + # no scheme to derive with off aws / gcp, template or not + _binding("dynamodb", "local", bucket=_TEMPLATE), + ], + ids=["non-env-template", "mixed-template", "local-platform"], +) +def test_a_bucket_that_cannot_derive_anywhere_stays_a_hard_error(unset, binding): + assert unset_bucket_env_vars(binding) == () + errors = validate_iceberg_sink(_contract(binding))[0] + assert any("dynamodb catalog requires binding.location.warehouse" in e for e in errors) + + +def test_an_override_warehouse_needs_no_bucket_warning(unset): + contract = _contract( + _TEMPLATED[0], + props={"iceberg_catalog_overrides": {"iceberg.catalog.warehouse": "s3://ops/wh/"}}, + ) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] and not [w for w in warnings if _VAR in w], warnings + assert iceberg_sink_preflight(contract, "ingest") is None + + +def test_glue_with_an_unresolved_template_is_unchanged(unset): + binding = _binding("glue", bucket=_TEMPLATE, region="eu-west-1") + contract = _contract(binding) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None + + +_CONTRACT_YAML = ( + """\ +fluidVersion: 0.7.6 +kind: DataProduct +domain: sales +metadata: + layer: Bronze + owner: + team: dp +id: bronze.o +name: o +exposes: +- exposeId: orders + kind: table + binding: + platform: aws + format: iceberg + location: + database: streaming + table: orders + catalog: dynamodb + bucket: '{{ env.%s }}' + region: eu-west-1 + contract: + schema: + - name: id + type: integer +builds: +- id: b1 + pattern: acquisition + engine: kafka-connect + outputs: + - orders + properties: + source: + kind: postgres + mode: incremental_append + streams: + - public.o + sink: + format: iceberg + kafka-connect: + deployment: + mode: bring-your-own + server_url: http://c:8083 +""" + % _VAR +) + + +@pytest.mark.parametrize("value", [None, "acme-lake"], ids=["unset", "set"]) +def test_cli_validate_accepts_the_contract_with_or_without_the_variable(tmp_path, value): + path = tmp_path / "kc_ddb_envbucket.fluid.yaml" + path.write_text(textwrap.dedent(_CONTRACT_YAML), encoding="utf-8") + env = os.environ.copy() + env.pop(_VAR, None) + if value is not None: + env[_VAR] = value + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + env["FLUID_NONINTERACTIVE"] = "1" + proc = subprocess.run( + [sys.executable, "-m", "fluid_build", "validate", str(path)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + timeout=120, + ) + out = proc.stdout + proc.stderr + assert proc.returncode == 0, out + assert "requires binding.location.warehouse" not in out + assert (_VAR in out) is (value is None), out From 41556ddaf6449b3c462b64f715326f5150afedf0 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:10:40 +0200 Subject: [PATCH 07/19] fix(iceberg): an empty bucket variable is named like an unset one A {{ env.* }} bucket whose variable is set but empty resolves to an empty bucket, which derives no warehouse. Validate warned only for unset variables, so the empty case fell through to the error that asks for the bucket the contract already sets. The check now reads the raw template and names every variable that is unset or empty. --- .../kafka_connect/iceberg_sink_validation.py | 4 +-- fluid_build/providers/_iceberg_catalog.py | 21 ++++++------ .../test_fw2_catalog_table_envbucket.py | 34 ++++++++++++++++--- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index a6f7d492..ab00e56d 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -503,10 +503,10 @@ def _check_catalog( def _unresolved_bucket_issue( bid: Any, kind: str, binding: Mapping[str, Any], names: Tuple[str, ...] ) -> str: - """Check 4's message for a warehouse that waits on unset bucket variables.""" + """Check 4's message for a warehouse that waits on unset or empty bucket variables.""" raw = (binding.get("location") or {}).get("bucket") one = len(names) == 1 - unset = f"{', '.join(names)} {'is' if one else 'are'} not set" + unset = f"{', '.join(names)} {'is' if one else 'are'} unset or empty" if _AT_RUN.get(): return ( f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 7c0ca345..51536d32 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -26,6 +26,7 @@ from __future__ import annotations import hashlib +import os import re from dataclasses import dataclass, field from types import MappingProxyType @@ -37,7 +38,6 @@ bucket_uses_fallback, get_iceberg_warehouse, normalize_location, - resolve_env_templates, ) # Apache Iceberg runtime class names (pinned to the connector surface validated @@ -502,13 +502,14 @@ def _bucket_warehouse(binding: Mapping[str, Any]) -> str: def unset_bucket_env_vars(binding: Mapping[str, Any]) -> Tuple[str, ...]: - """The unset variables an explicit bucket's derived warehouse waits on, or ``()``. + """The unset or empty variables in an explicit bucket's ``{{ env.* }}`` templates. Non-empty when ``location.bucket`` is set on a platform - :func:`_bucket_warehouse` derives on (aws, gcp) but its ``{{ env.* }}`` - templates do not resolve in this process's environment: the bucket is - explicit, so the warehouse derives wherever those variables are set, but - not here. ``()`` for a resolved or absent bucket, another platform, or a + :func:`_bucket_warehouse` derives on (aws, gcp) and names ``{{ env.* }}`` + variables that are unset or empty in this process's environment: the + bucket is explicit, so the warehouse derives wherever those variables are + set, but not here. ``()`` when every variable the bucket names has a + value, for an absent bucket, another platform, or a bucket holding a template that is not ``{{ env.* }}`` (nothing resolves that one). """ from ..iac.provider_match import canonical_cloud @@ -517,11 +518,11 @@ def unset_bucket_env_vars(binding: Mapping[str, Any]) -> Tuple[str, ...]: raw = loc.get("bucket") if not raw or canonical_cloud(binding.get("platform")) not in _BUCKET_WAREHOUSE_SCHEMES: return () - resolved = str(resolve_env_templates(raw)) - names = tuple(dict.fromkeys(_ENV_TEMPLATE_RE.findall(resolved))) - if not names or "{{" in _ENV_TEMPLATE_RE.sub("", resolved): + raw = str(raw) + if "{{" in _ENV_TEMPLATE_RE.sub("", raw): return () - return names + found = (name.strip() for name in _ENV_TEMPLATE_RE.findall(raw)) + return tuple(dict.fromkeys(name for name in found if not os.environ.get(name))) # --------------------------------------------------------------------------- diff --git a/tests/build_runners/test_fw2_catalog_table_envbucket.py b/tests/build_runners/test_fw2_catalog_table_envbucket.py index 0e695217..b8026875 100644 --- a/tests/build_runners/test_fw2_catalog_table_envbucket.py +++ b/tests/build_runners/test_fw2_catalog_table_envbucket.py @@ -99,7 +99,7 @@ def test_unresolved_bucket_template_warns_at_validate(unset, binding, engine): assert len(hit) == 1, warnings assert f"the {kind} catalog's warehouse derives from binding.location.bucket" in hit[0] assert repr(_TEMPLATE) in hit[0] - assert f"{_VAR} is not set here" in hit[0] + assert f"{_VAR} is unset or empty here" in hit[0] # It does not ask for the bucket the contract already sets. assert "requires binding.location.warehouse" not in hit[0] @@ -109,7 +109,7 @@ def test_unresolved_bucket_template_warns_at_validate(unset, binding, engine): def test_unresolved_bucket_template_is_refused_by_the_preflight(unset, binding, engine): refusal = iceberg_sink_preflight(_contract(binding, engine=engine), "ingest") assert refusal is not None - assert f"{_VAR} is not set in the runner's environment" in refusal + assert f"{_VAR} is unset or empty in the runner's environment" in refusal assert "binding.location.warehouse" in refusal @@ -150,7 +150,31 @@ def test_several_unset_variables_are_all_named(monkeypatch): binding = _binding("dynamodb", bucket="{{ env.FW2_A }}-{{ env.FW2_B }}", region="eu-west-1") assert unset_bucket_env_vars(binding) == ("FW2_A", "FW2_B") warnings = validate_iceberg_sink(_contract(binding))[1] - assert any("FW2_A, FW2_B are not set here" in w and "Set them" in w for w in warnings) + assert any("FW2_A, FW2_B are unset or empty here" in w and "Set them" in w for w in warnings) + + +def test_only_the_variables_without_a_value_are_named(monkeypatch): + monkeypatch.setenv("FW2_A", "acme") + monkeypatch.delenv("FW2_B", raising=False) + binding = _binding("dynamodb", bucket="{{ env.FW2_B }}", region="eu-west-1") + assert unset_bucket_env_vars(binding) == ("FW2_B",) + binding = _binding("dynamodb", bucket="{{ env.FW2_A }}", region="eu-west-1") + assert unset_bucket_env_vars(binding) == () + + +@pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) +def test_an_empty_variable_is_treated_like_an_unset_one(monkeypatch, binding): + # An empty value resolves the bucket to "", which derives no warehouse; the + # message still names the variable instead of asking for a bucket. + monkeypatch.setenv(_VAR, "") + contract = _contract(binding) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert any(f"{_VAR} is unset or empty here" in w for w in warnings), warnings + refusal = iceberg_sink_preflight(contract, "ingest") + assert refusal is not None + assert f"{_VAR} is unset or empty in the runner's environment" in refusal + assert "requires binding.location.warehouse" not in refusal @pytest.mark.parametrize( @@ -237,7 +261,7 @@ def test_glue_with_an_unresolved_template_is_unchanged(unset): ) -@pytest.mark.parametrize("value", [None, "acme-lake"], ids=["unset", "set"]) +@pytest.mark.parametrize("value", [None, "", "acme-lake"], ids=["unset", "empty", "set"]) def test_cli_validate_accepts_the_contract_with_or_without_the_variable(tmp_path, value): path = tmp_path / "kc_ddb_envbucket.fluid.yaml" path.write_text(textwrap.dedent(_CONTRACT_YAML), encoding="utf-8") @@ -258,4 +282,4 @@ def test_cli_validate_accepts_the_contract_with_or_without_the_variable(tmp_path out = proc.stdout + proc.stderr assert proc.returncode == 0, out assert "requires binding.location.warehouse" not in out - assert (_VAR in out) is (value is None), out + assert (_VAR in out) is (not value), out From e3fb8cd5b9c61a7a3d3f6a1dd6d8bd7c11f95eb4 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:40:01 +0200 Subject: [PATCH 08/19] fix(iceberg): the sink preflight reads an env-templated bucket as written The acquisition dispatcher resolves {{ env.X }} before the runner sees the contract, turning an unset or empty X into an empty string. The preflight then passed acme-{{ env.LAKE_ENV }}-lake as acme--lake and the sink was pushed a warehouse in a bucket the contract does not name. The dispatcher now runs the streaming runners inside contract_as_written, so the preflight reads the bucket template, names the variable and refuses the build. The bigquery deriver renders the bucket's env templates and derives no warehouse from one that does not resolve; an empty variable counts as unset for every bucket-derived warehouse. The Debezium bigquery no-warehouse warning says the server refuses to boot, since debezium.sink.iceberg.warehouse has no default. --- fluid_build/build_runners/base.py | 25 +- .../kafka_connect/iceberg_sink_validation.py | 155 ++++++-- fluid_build/providers/_iceberg_catalog.py | 83 +++-- .../test_fw2_catalog_table_dispatch.py | 332 ++++++++++++++++++ .../test_fw2_catalog_table_envbucket.py | 8 +- .../test_fw2_catalog_table_overrides.py | 10 +- 6 files changed, 538 insertions(+), 75 deletions(-) create mode 100644 tests/build_runners/test_fw2_catalog_table_dispatch.py diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index e91f80bc..7b9f55c8 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -252,21 +252,32 @@ def _execute_acquisition_build( # everything (including secret-shaped vars). Catalog-export paths use # ``cli/_common.py::resolve_contract_env_templates`` instead, which # leaves sensitive placeholders literal. + unresolved_contract = contract build = _resolve_env_placeholders(build) contract = _resolve_env_placeholders(contract) module_path, function_name = entry + import contextlib import importlib module = importlib.import_module(module_path, package="fluid_build.build_runners") runner_fn = getattr(module, function_name) - return runner_fn( - build, - contract, - contract_dir, - dry_run=dry_run, - sample_rows=sample_rows, - ) + # The Iceberg sink preflight reads a bucket's ``{{ env.X }}`` template as + # written: the resolution above turned an unset ``X`` into "", which would + # point the sink at a bucket the contract does not name. + scope: contextlib.AbstractContextManager[None] = contextlib.nullcontext() + if engine in ("kafka-connect", "debezium"): + from .kafka_connect.iceberg_sink_validation import contract_as_written + + scope = contract_as_written(unresolved_contract) + with scope: + return runner_fn( + build, + contract, + contract_dir, + dry_run=dry_run, + sample_rows=sample_rows, + ) def _execute_embedded_sql_build_snowflake( diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index ab00e56d..6000a0ec 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -45,9 +45,11 @@ hand-written config selects (its ``type`` or ``catalog-impl``); * every ``binding.location`` key in the row's ``sink_requires`` must be set, a warehouse counting when the deriver resolves one (DynamoDB and JDBC derive - it from an explicit bucket; a bucket ``{{ env.* }}`` template that does not - resolve is a warning here and an error in the runner's preflight); Glue - keeps its advisory region warning: the warehouse falls back; + it from an explicit bucket); Glue keeps its advisory region warning: the + warehouse falls back; +* a bucket ``{{ env.* }}`` template the DynamoDB, JDBC or BigQuery warehouse + derives from, naming a variable that is unset or empty, is a warning here + and an error in the runner's preflight; * the runtime must ship the catalog's client (the stock Apache Iceberg Kafka Connect runtime has no Nessie client, and the published sink predates the ``bigquery`` catalog type); @@ -73,10 +75,22 @@ from __future__ import annotations +import contextlib import contextvars import logging from dataclasses import dataclass -from typing import AbstractSet, Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from typing import ( + AbstractSet, + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, +) from ...providers._iceberg_catalog import ( BIGQUERY_PROJECT_ID, @@ -249,12 +263,15 @@ def iceberg_sink_preflight( never fatal, the same split as ``fluid validate`` without ``--strict``, with one difference: a ``location.bucket`` template that ``fluid validate`` only warns about (it may resolve where the sink runs) is an error here, - because the runner derives the sink config in this process next. + because the runner derives the sink config in this process next. The + template is read from the contract as written when the runner runs inside + :func:`contract_as_written`, as the acquisition dispatcher runs it. """ build = executing_build(contract, build_id) if build is None: return None - token = _AT_RUN.set(True) + run_only: List[str] = [] + token = _AT_RUN.set(run_only) try: errors, warnings = _validate(contract, builds=(build,)) finally: @@ -263,7 +280,9 @@ def iceberg_sink_preflight( (log or LOG).warning("iceberg_sink.preflight.warning build=%s %s", build_id, msg) if not errors: return None - return "iceberg sink preflight failed (see `fluid validate`): " + "; ".join(errors) + # ``fluid validate`` reports every error but the run-only ones. + see = " (see `fluid validate`)" if any(e not in run_only for e in errors) else "" + return f"iceberg sink preflight failed{see}: " + "; ".join(errors) def _validate(contract: Mapping[str, Any], *, builds: Optional[Iterable[Any]] = None) -> _Issues: @@ -403,21 +422,32 @@ def _check_catalog( # into the config it pushes. A bucket whose ``{{ env.* }}`` template # does not resolve here is explicit, so ``fluid validate`` only warns: # the runner's preflight, in the environment the sink config is - # derived in, refuses it. Glue requires nothing (the warehouse falls - # back), so its region stays advisory. + # derived in, refuses it (BigQuery's too, which needs no warehouse to + # start: the alternative is a warehouse in a bucket the contract does + # not name). Glue requires nothing (the warehouse falls back), so its + # region stays advisory. from ...providers._iceberg_catalog import unset_bucket_env_vars derived_warehouse = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse + as_written = _binding_as_written(binding) + waits_on = () if _overrides_warehouse(runtime) else unset_bucket_env_vars(as_written, kind) + if waits_on: + required = "warehouse" in info.sink_requires + issue = _unresolved_bucket_issue(bid, kind, as_written, waits_on, required=required) + run_only = _AT_RUN.get() + if run_only is None: + warnings.append(issue) + else: + errors.append(issue) + run_only.append(issue) for key in info.sink_requires: + if key == "warehouse" and waits_on: + continue # reported above, naming the variables present = derived_warehouse if key == "warehouse" else loc.get(key) start_prop = _start_only_property(key, kind) if not present and start_prop is not None: present = _overrides_catalog_property(runtime, start_prop) - unset = unset_bucket_env_vars(binding) if key == "warehouse" else () - if not present and unset and kind in BUCKET_WAREHOUSE_KINDS: - issue = _unresolved_bucket_issue(bid, kind, binding, unset) - (errors if _AT_RUN.get() else warnings).append(issue) - elif not present: + if not present: errors.append( f"iceberg sink (build {bid!r}): {kind} catalog requires " f"binding.location.{key}{_requires_hint(key, kind, info)}" @@ -459,15 +489,20 @@ def _check_catalog( # :298-303). The Kafka Connect sink's auto-create calls createNamespace # before every createTable and catches only AlreadyExists / Forbidden # (IcebergWriterFactory.java:89, :122-137), so every table auto-create - # throws, even in a dataset that exists. The Debezium Server sink creates - # a missing table, and its dataset only when the dataset is missing - # (memiiso/debezium-server-iceberg 1.2.0.Final IcebergChangeConsumer.java - # :322-325, IcebergUtil.java:98-126). Advisory: tables that exist need none. + # throws, even in a dataset that exists. Debezium Server needs one to boot: + # IcebergConfig declares ``debezium.sink.iceberg.warehouse`` as a String + # with no default (memiiso/debezium-server-iceberg 1.2.0.Final + # IcebergConfig.java:44-45), in the config mapping GlobalConfig nests + # (GlobalConfig.java:14-15) and IcebergChangeConsumer injects + # (IcebergChangeConsumer.java:79), and SmallRye Config refuses to start + # with a required mapping property missing. Advisory; a bucket that waits + # on a variable is reported above. if ( reaching == "bigquery" and kind == "bigquery" and runtime.derives and not derived_warehouse + and not waits_on and not _overrides_warehouse(runtime) ): warnings.append( @@ -475,11 +510,11 @@ def _check_catalog( "binding.location (a gs:// warehouse, or a bucket), so the derived bigquery " "catalog config sets none. " + ( - "The Debezium Server Iceberg sink creates a missing table, which " - "BigQueryMetastoreCatalog refuses without a warehouse unless the table's " - "dataset exists with a default storage location URI (a missing dataset it " - "refuses outright); tables that exist need none. The sink also declares " - "debezium.sink.iceberg.warehouse with no default" + "Debezium Server declares debezium.sink.iceberg.warehouse with no default " + "(debezium-server-iceberg 1.2.0.Final IcebergConfig), so the server refuses " + "to boot without it, whether or not the tables exist. Set " + "binding.location.warehouse or binding.location.bucket, or warehouse in " + "debezium.server.sink.config" if runtime.engine == "debezium" else "With iceberg.tables.auto-create-enabled the sink calls createNamespace " "for every table it creates, and BigQueryMetastoreCatalog.createNamespace " @@ -492,35 +527,85 @@ def _check_catalog( _check_selector_overrides(bid, kind, info, runtime, errors) -#: True while :func:`iceberg_sink_preflight` runs. The runner derives the sink -#: config in the same process right after it, so a ``location.bucket`` -#: template this environment cannot resolve leaves the sink with no warehouse. -_AT_RUN: contextvars.ContextVar[bool] = contextvars.ContextVar( - "iceberg_sink_preflight", default=False +#: The run-only errors of the :func:`iceberg_sink_preflight` that is running, +#: ``None`` under ``fluid validate``. The runner derives the sink config in +#: the same process right after the preflight, so a ``location.bucket`` +#: template this environment cannot resolve leaves the sink with no +#: warehouse, or one in a bucket the contract does not name. +_AT_RUN: contextvars.ContextVar[Optional[List[str]]] = contextvars.ContextVar( + "iceberg_sink_preflight", default=None +) + +#: The contract as written, before the acquisition dispatcher resolved its +#: ``{{ env.* }}`` templates (see :func:`contract_as_written`). +_AS_WRITTEN: contextvars.ContextVar[Optional[Mapping[str, Any]]] = contextvars.ContextVar( + "iceberg_sink_contract_as_written", default=None ) +@contextlib.contextmanager +def contract_as_written(contract: Mapping[str, Any]) -> Iterator[None]: + """Let :func:`iceberg_sink_preflight` read ``contract`` as written. + + ``base._execute_acquisition_build`` resolves every ``{{ env.X }}`` before + the runner sees the contract, and a missing or empty ``X`` becomes ``""``: + ``acme-{{ env.LAKE_ENV }}-lake`` reaches the runner as ``acme--lake``. + Inside this block the preflight reads the bucket template from + ``contract``, so it names the variable and refuses the build instead of + passing a warehouse in a bucket the contract does not name. + """ + token = _AS_WRITTEN.set(contract) + try: + yield + finally: + _AS_WRITTEN.reset(token) + + +def _binding_as_written(binding: Mapping[str, Any]) -> Mapping[str, Any]: + """``binding`` as the contract wrote it, while a runner runs; else ``binding``. + + The expose binding in the contract :func:`contract_as_written` holds whose + resolution by the dispatcher's own resolver is ``binding``. + """ + contract = _AS_WRITTEN.get() + if contract is None or _AT_RUN.get() is None: + return binding + from ..base import _resolve_env_placeholders + + for expose in contract.get("exposes") or []: + raw = expose.get("binding") if isinstance(expose, Mapping) else None + if isinstance(raw, Mapping) and _resolve_env_placeholders(dict(raw)) == binding: + return raw + return binding + + def _unresolved_bucket_issue( - bid: Any, kind: str, binding: Mapping[str, Any], names: Tuple[str, ...] + bid: Any, + kind: str, + binding: Mapping[str, Any], + names: Tuple[str, ...], + *, + required: bool, ) -> str: """Check 4's message for a warehouse that waits on unset or empty bucket variables.""" raw = (binding.get("location") or {}).get("bucket") one = len(names) == 1 unset = f"{', '.join(names)} {'is' if one else 'are'} unset or empty" - if _AT_RUN.get(): + refuses = f"the {kind} catalog refuses to start without a warehouse" + if _AT_RUN.get() is not None: return ( f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " f"binding.location.bucket {raw!r}, and {unset} in the runner's environment, so " - f"the sink would get no warehouse and the {kind} catalog refuses to start " - f"without one. Set {'it' if one else 'them'} here, or set " - "binding.location.warehouse" + "the bucket does not resolve: the sink would get a warehouse in a bucket the " + f"contract does not name, or none{f', and {refuses}' if required else ''}. " + f"Set {'it' if one else 'them'} here, or set binding.location.warehouse" ) return ( f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " f"binding.location.bucket {raw!r}, and {unset} here, so it cannot be derived at " f"validate time. Set {'it' if one else 'them'} where the sink runs: the runner " - "refuses the build when the bucket does not resolve there, because the " - f"{kind} catalog refuses to start without a warehouse" + "refuses the build when the bucket does not resolve there" + + (f", because {refuses}" if required else "") ) diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 51536d32..4898b104 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -35,9 +35,9 @@ from ._sql_safety import validate_ident from .aws.util.warehouse import ( _ENV_TEMPLATE_RE, - bucket_uses_fallback, get_iceberg_warehouse, normalize_location, + resolve_env_templates, ) # Apache Iceberg runtime class names (pinned to the connector surface validated @@ -420,7 +420,8 @@ def resolve_iceberg_catalog( ``uri`` + ``warehouse`` (catalog name) and don't use it. DynamoDB and JDBC take ``location.warehouse``, else derive one from an explicit ``location.bucket`` (:data:`BUCKET_WAREHOUSE_KINDS`); BigQuery takes the - ``gs://`` URI :func:`iceberg_storage_uri` gives, plus ``location.project`` + ``gs://`` URI :func:`iceberg_storage_uri` gives for the bucket with its + ``{{ env.* }}`` templates rendered, plus ``location.project`` and ``location.region`` as its own catalog properties. """ loc = binding.get("location") or {} @@ -456,9 +457,12 @@ def resolve_iceberg_catalog( extra: Dict[str, str] = {} if kind == "bigquery": # The gs:// storage dbt-bigquery's ``external_volume`` and the GCP IaC - # use, or none. ``client.region`` is an AWS client property, so the - # region goes to the catalog's own location key instead. - warehouse = iceberg_storage_uri(binding, scheme="gs") + # use, or none, with the bucket's ``{{ env.* }}`` templates rendered: + # a bucket that does not resolve here derives none. ``client.region`` + # is an AWS client property, so the region goes to the catalog's own + # location key instead. + rendered = {**loc, "bucket": _rendered_bucket(loc)} + warehouse = iceberg_storage_uri({"location": rendered}, scheme="gs") if loc.get("project"): extra[BIGQUERY_PROJECT_ID] = str(loc["project"]) if region: @@ -488,41 +492,70 @@ def _bucket_warehouse(binding: Mapping[str, Any]) -> str: the Glue row's :func:`get_iceberg_warehouse` normalises them (``path`` defaults to ``/
/``). The scheme follows the platform: ``s3`` on AWS, ``gs`` on GCP; any other platform derives nothing. A - missing or unresolved bucket derives nothing either, rather than the - Glue row's account-derived fallback bucket. + missing bucket, or one :func:`_rendered_bucket` cannot render, derives + nothing either, rather than the Glue row's account-derived fallback bucket. """ from ..iac.provider_match import canonical_cloud loc = binding.get("location") or {} scheme = _BUCKET_WAREHOUSE_SCHEMES.get(canonical_cloud(binding.get("platform"))) - if not scheme or bucket_uses_fallback(loc): + bucket = _rendered_bucket(loc) + if not scheme or not bucket: return "" - bucket, path = normalize_location(loc, account_ref="") + _bucket, path = normalize_location(loc, account_ref="") return f"{scheme}://{bucket}/{path}" -def unset_bucket_env_vars(binding: Mapping[str, Any]) -> Tuple[str, ...]: - """The unset or empty variables in an explicit bucket's ``{{ env.* }}`` templates. +def _rendered_bucket(loc: Mapping[str, Any]) -> str: + """``location.bucket`` with its ``{{ env.* }}`` templates rendered, or ``""``. - Non-empty when ``location.bucket`` is set on a platform - :func:`_bucket_warehouse` derives on (aws, gcp) and names ``{{ env.* }}`` - variables that are unset or empty in this process's environment: the - bucket is explicit, so the warehouse derives wherever those variables are - set, but not here. ``()`` when every variable the bucket names has a - value, for an absent bucket, another platform, or a bucket holding a - template that is not ``{{ env.* }}`` (nothing resolves that one). + ``""`` when the bucket is absent, names a variable that is unset or empty + in this process's environment, or holds a template that is not + ``{{ env.* }}``. An empty variable counts as unset: rendering it would + turn ``acme-{{ env.LAKE_ENV }}-lake`` into ``acme--lake``, a bucket the + contract does not name. + """ + raw = str(loc.get("bucket") or "").strip() + if not raw or _unset_env_vars(raw) or "{{" in _ENV_TEMPLATE_RE.sub("", raw): + return "" + return str(resolve_env_templates(raw)).strip() + + +def _unset_env_vars(raw: str) -> Tuple[str, ...]: + """The ``{{ env.* }}`` variables ``raw`` names that are unset or empty here.""" + found = (name.strip() for name in _ENV_TEMPLATE_RE.findall(raw)) + return tuple(dict.fromkeys(name for name in found if not os.environ.get(name))) + + +def unset_bucket_env_vars(binding: Mapping[str, Any], kind: str) -> Tuple[str, ...]: + """The unset or empty variables the ``kind`` sink's warehouse waits on. + + Non-empty when the sink resolver derives ``kind``'s warehouse from + ``location.bucket`` and that bucket names ``{{ env.* }}`` variables that + are unset or empty in this process's environment: DynamoDB and JDBC on + aws or gcp with no ``location.warehouse``, and BigQuery with no + scheme-qualified ``location.warehouse``. The bucket is explicit, so the + warehouse derives wherever those variables are set, but not here. ``()`` + when every variable the bucket names has a value, for an absent bucket, + another kind or platform, a warehouse the binding sets, or a bucket + holding a template that is not ``{{ env.* }}`` (nothing resolves that one). """ from ..iac.provider_match import canonical_cloud loc = binding.get("location") or {} - raw = loc.get("bucket") - if not raw or canonical_cloud(binding.get("platform")) not in _BUCKET_WAREHOUSE_SCHEMES: - return () - raw = str(raw) - if "{{" in _ENV_TEMPLATE_RE.sub("", raw): + raw = str(loc.get("bucket") or "") + warehouse = str(loc.get("warehouse") or "").strip() + if kind in BUCKET_WAREHOUSE_KINDS: + reads_bucket = not warehouse and ( + canonical_cloud(binding.get("platform")) in _BUCKET_WAREHOUSE_SCHEMES + ) + else: + # :func:`iceberg_storage_uri` reads the bucket unless the warehouse + # carries a scheme. + reads_bucket = kind == "bigquery" and not warehouse.startswith(_WAREHOUSE_SCHEMES) + if not raw or not reads_bucket or "{{" in _ENV_TEMPLATE_RE.sub("", raw): return () - found = (name.strip() for name in _ENV_TEMPLATE_RE.findall(raw)) - return tuple(dict.fromkeys(name for name in found if not os.environ.get(name))) + return _unset_env_vars(raw) # --------------------------------------------------------------------------- diff --git a/tests/build_runners/test_fw2_catalog_table_dispatch.py b/tests/build_runners/test_fw2_catalog_table_dispatch.py new file mode 100644 index 00000000..1df1bcc6 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_dispatch.py @@ -0,0 +1,332 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An env-templated bucket, through the acquisition dispatcher a run goes through. + +``base._execute_acquisition_build`` resolves every ``{{ env.X }}`` before the +runner sees the contract, and a missing or empty ``X`` becomes ``""``. The +Iceberg sink preflight read only that resolved contract, so +``acme-{{ env.LAKE_ENV }}-lake`` with ``LAKE_ENV`` unset reached it as +``acme--lake``: the preflight passed and the sink was pushed a warehouse in a +bucket the contract does not name. A fully templated bucket reached it as +``""`` and was refused with a message asking for the bucket the contract sets. +The BigQuery deriver did not render the template at all, and pushed a literal +``gs://{{ env.DATA_LAKE_BUCKET }}`` warehouse even with the variable set. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +from fluid_build.build_runners import base +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + +_VAR = "FW2_LAKE_ENV" +_FULL = "{{ env.%s }}" % _VAR +_PARTIAL = "acme-{{ env.%s }}-lake" % _VAR + + +def _binding(catalog: str, bucket: str, **location: Any) -> Dict[str, Any]: + platform = "gcp" if catalog == "bigquery" else "aws" + loc: Dict[str, Any] = {"database": "streaming", "table": "orders", "catalog": catalog} + loc["bucket"] = bucket + if catalog == "bigquery": + loc["project"] = "acme-proj" + else: + loc["region"] = "eu-west-1" + loc.update(location) + return {"platform": platform, "format": "iceberg", "location": loc} + + +def _contract(binding: Dict[str, Any], engine: str) -> Dict[str, Any]: + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": binding, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + ], + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": ["orders"], + "properties": props, + } + ], + } + + +class _FakeConnect: + """Records the connector configs the Kafka Connect runner pushes.""" + + pushed: List[Tuple[str, Dict[str, Any]]] = [] + + def __init__(self, *_a: Any, **_k: Any) -> None: + pass + + def get_connector(self, _name: str) -> None: + return None + + def create_connector(self, name: str, cfg: Dict[str, Any]) -> None: + type(self).pushed.append((name, dict(cfg))) + + update_config = create_connector + + def __getattr__(self, _name: str) -> Any: + return lambda *a, **k: {"connector": {"state": "RUNNING"}, "tasks": [{"state": "RUNNING"}]} + + +@pytest.fixture +def fake_connect(monkeypatch): + from fluid_build.build_runners.kafka_connect import runner as kc_runner + + _FakeConnect.pushed = [] + monkeypatch.setattr(kc_runner, "KafkaConnectRestClient", _FakeConnect) + return _FakeConnect.pushed + + +def _set(monkeypatch, value: Optional[str]) -> None: + if value is None: + monkeypatch.delenv(_VAR, raising=False) + else: + monkeypatch.setenv(_VAR, value) + + +def _dispatch(contract: Dict[str, Any], tmp_path: Path, monkeypatch) -> int: + monkeypatch.chdir(tmp_path) + return base._execute_acquisition_build(contract["builds"][0], contract, tmp_path, dry_run=False) + + +def _sink_warehouse(pushed: List[Tuple[str, Dict[str, Any]]]) -> Optional[str]: + sinks = [cfg for name, cfg in pushed if name.endswith("-sink")] + assert len(sinks) == 1, pushed + return sinks[0].get("iceberg.catalog.warehouse") + + +def _props_path(tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / "bronze.o" / "b1" / "application.properties" + + +def _preflight_failure(caplog) -> str: + # The runner returns 1; the message is in its error log. + text = "\n".join(r.getMessage() for r in caplog.records) + assert "iceberg sink preflight failed" in text, text + return text + + +# --------------------------------------------------------------------------- +# Kafka Connect, through the dispatcher +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc", "bigquery"]) +@pytest.mark.parametrize("bucket", [_FULL, _PARTIAL], ids=["full", "partial"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_dispatch_refuses_a_bucket_that_does_not_resolve( + fake_connect, tmp_path, monkeypatch, caplog, catalog, bucket, value +): + _set(monkeypatch, value) + extra = {"uri": "jdbc:postgresql://pg:5432/iceberg"} if catalog == "jdbc" else {} + contract = _contract(_binding(catalog, bucket, **extra), "kafka-connect") + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 1 + # Nothing reached the cluster: no sink in another bucket, no source. + assert fake_connect == [] + text = _preflight_failure(caplog) + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert repr(bucket) in text + assert "requires binding.location.warehouse" not in text + # `fluid validate` only warns about this, so the refusal does not send + # the operator there. + assert "see `fluid validate`" not in text + + +@pytest.mark.parametrize( + "catalog, bucket, expected", + [ + ("dynamodb", _PARTIAL, "s3://acme-prod-lake/streaming/orders/"), + ("dynamodb", _FULL, "s3://prod/streaming/orders/"), + ("bigquery", _PARTIAL, "gs://acme-prod-lake"), + ("bigquery", _FULL, "gs://prod"), + ], + ids=["ddb-partial", "ddb-full", "bq-partial", "bq-full"], +) +def test_kc_dispatch_derives_from_the_resolved_bucket( + fake_connect, tmp_path, monkeypatch, catalog, bucket, expected +): + _set(monkeypatch, "prod") + rc = _dispatch(_contract(_binding(catalog, bucket), "kafka-connect"), tmp_path, monkeypatch) + assert rc == 0 + assert _sink_warehouse(fake_connect) == expected + + +def test_kc_dispatch_a_set_warehouse_ignores_the_bucket_variable( + fake_connect, tmp_path, monkeypatch +): + _set(monkeypatch, None) + binding = _binding("dynamodb", _PARTIAL, warehouse="s3://ops/wh/") + assert _dispatch(_contract(binding, "kafka-connect"), tmp_path, monkeypatch) == 0 + assert _sink_warehouse(fake_connect) == "s3://ops/wh/" + + +def test_the_dispatcher_scope_ends_with_the_run(fake_connect, tmp_path, monkeypatch): + # Outside the dispatcher the preflight reads the contract it is handed. + _set(monkeypatch, None) + contract = _contract(_binding("dynamodb", _PARTIAL), "kafka-connect") + assert _dispatch(contract, tmp_path, monkeypatch) == 1 + resolved = base._resolve_env_placeholders(contract) + assert iceberg_sink_preflight(resolved, "b1") is None + + +# --------------------------------------------------------------------------- +# Debezium Server (embedded), through the dispatcher +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bucket", [_FULL, _PARTIAL], ids=["full", "partial"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_debezium_dispatch_refuses_a_bucket_that_does_not_resolve( + tmp_path, monkeypatch, caplog, bucket, value +): + _set(monkeypatch, value) + contract = _contract(_binding("dynamodb", bucket), "debezium") + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 1 + assert not _props_path(tmp_path).exists() + text = _preflight_failure(caplog) + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert "acme--lake" not in text + + +def test_debezium_dispatch_derives_from_the_resolved_bucket(tmp_path, monkeypatch): + _set(monkeypatch, "prod") + # No server binary on PATH: the runner fails after writing the config. + _dispatch(_contract(_binding("dynamodb", _PARTIAL), "debezium"), tmp_path, monkeypatch) + text = _props_path(tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://acme-prod-lake/streaming/orders/" in text + + +# --------------------------------------------------------------------------- +# BigQuery: the deriver renders the bucket, in all three states +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value, warehouse", + [("mylake", "gs://mylake"), ("", ""), (None, "")], + ids=["set", "empty", "unset"], +) +def test_bigquery_env_bucket_derive_and_validate(monkeypatch, value, warehouse): + _set(monkeypatch, value) + binding = _binding("bigquery", _FULL) + resolved = resolve_iceberg_catalog(binding) + assert resolved.warehouse == warehouse + assert "{{" not in resolved.warehouse + assert resolved.extra_catalog_props["gcp.bigquery.project-id"] == "acme-proj" + + contract = _contract(binding, "kafka-connect") + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + named = [w for w in warnings if _VAR in w] + refusal = iceberg_sink_preflight(contract, "b1") + if value: + assert named == [] and refusal is None + assert not [w for w in warnings if "no gs:// warehouse" in w], warnings + else: + # One warning naming the variable, not the generic no-warehouse one. + assert len(named) == 1, warnings + assert f"{_VAR} is unset or empty here" in named[0] + assert not [w for w in warnings if "no gs:// warehouse" in w], warnings + assert refusal is not None + assert f"{_VAR} is unset or empty in the runner's environment" in refusal + + +def test_bigquery_partial_bucket_with_an_empty_variable_derives_none(monkeypatch): + _set(monkeypatch, "") + assert resolve_iceberg_catalog(_binding("bigquery", _PARTIAL)).warehouse == "" + + +def test_bigquery_gs_warehouse_wins_over_an_unresolved_bucket(monkeypatch): + _set(monkeypatch, None) + binding = _binding("bigquery", _FULL, warehouse="gs://wh/x") + assert resolve_iceberg_catalog(binding).warehouse == "gs://wh/x" + errors, warnings = validate_iceberg_sink(_contract(binding, "kafka-connect")) + assert errors == [] and not [w for w in warnings if _VAR in w], warnings + + +def test_dynamodb_partial_bucket_with_an_empty_variable_derives_none(monkeypatch): + # Rendering it would give s3://acme--lake/..., a bucket the contract does + # not name; validate warns and names the variable instead. + _set(monkeypatch, "") + binding = _binding("dynamodb", _PARTIAL) + assert resolve_iceberg_catalog(binding).warehouse == "" + errors, warnings = validate_iceberg_sink(_contract(binding, "kafka-connect")) + assert errors == [] + assert any(f"{_VAR} is unset or empty here" in w for w in warnings), warnings + + +# --------------------------------------------------------------------------- +# Debezium BigQuery with no warehouse: the server does not boot +# --------------------------------------------------------------------------- + + +def test_debezium_bigquery_no_warehouse_warning_says_the_server_refuses_to_boot(): + binding = _binding("bigquery", "") + binding["location"].pop("bucket") + errors, warnings = validate_iceberg_sink(_contract(binding, "debezium")) + assert errors == [] + hit = [w for w in warnings if "no gs:// warehouse" in w] + assert len(hit) == 1, warnings + assert "refuses to boot" in hit[0] + assert "debezium.sink.iceberg.warehouse" in hit[0] + assert "tables that exist need none" not in hit[0] + + +def test_kafka_connect_bigquery_no_warehouse_warning_is_unchanged(): + binding = _binding("bigquery", "") + binding["location"].pop("bucket") + warnings = validate_iceberg_sink(_contract(binding, "kafka-connect"))[1] + hit = [w for w in warnings if "no gs:// warehouse" in w] + assert len(hit) == 1 and "without auto-create, tables that exist need none" in hit[0] diff --git a/tests/build_runners/test_fw2_catalog_table_envbucket.py b/tests/build_runners/test_fw2_catalog_table_envbucket.py index b8026875..ef883761 100644 --- a/tests/build_runners/test_fw2_catalog_table_envbucket.py +++ b/tests/build_runners/test_fw2_catalog_table_envbucket.py @@ -148,7 +148,7 @@ def test_several_unset_variables_are_all_named(monkeypatch): monkeypatch.delenv("FW2_A", raising=False) monkeypatch.delenv("FW2_B", raising=False) binding = _binding("dynamodb", bucket="{{ env.FW2_A }}-{{ env.FW2_B }}", region="eu-west-1") - assert unset_bucket_env_vars(binding) == ("FW2_A", "FW2_B") + assert unset_bucket_env_vars(binding, "dynamodb") == ("FW2_A", "FW2_B") warnings = validate_iceberg_sink(_contract(binding))[1] assert any("FW2_A, FW2_B are unset or empty here" in w and "Set them" in w for w in warnings) @@ -157,9 +157,9 @@ def test_only_the_variables_without_a_value_are_named(monkeypatch): monkeypatch.setenv("FW2_A", "acme") monkeypatch.delenv("FW2_B", raising=False) binding = _binding("dynamodb", bucket="{{ env.FW2_B }}", region="eu-west-1") - assert unset_bucket_env_vars(binding) == ("FW2_B",) + assert unset_bucket_env_vars(binding, "dynamodb") == ("FW2_B",) binding = _binding("dynamodb", bucket="{{ env.FW2_A }}", region="eu-west-1") - assert unset_bucket_env_vars(binding) == () + assert unset_bucket_env_vars(binding, "dynamodb") == () @pytest.mark.parametrize("binding", _TEMPLATED, ids=_IDS) @@ -189,7 +189,7 @@ def test_an_empty_variable_is_treated_like_an_unset_one(monkeypatch, binding): ids=["non-env-template", "mixed-template", "local-platform"], ) def test_a_bucket_that_cannot_derive_anywhere_stays_a_hard_error(unset, binding): - assert unset_bucket_env_vars(binding) == () + assert unset_bucket_env_vars(binding, "dynamodb") == () errors = validate_iceberg_sink(_contract(binding))[0] assert any("dynamodb catalog requires binding.location.warehouse" in e for e in errors) diff --git a/tests/build_runners/test_fw2_catalog_table_overrides.py b/tests/build_runners/test_fw2_catalog_table_overrides.py index b125cbf5..a43b3a57 100644 --- a/tests/build_runners/test_fw2_catalog_table_overrides.py +++ b/tests/build_runners/test_fw2_catalog_table_overrides.py @@ -223,11 +223,13 @@ def test_kafka_connect_warning_says_auto_create_fails_even_in_an_existing_datase assert "default storage location URI" not in warning -def test_debezium_warning_keeps_the_dataset_storage_uri_exception(): - # memiiso/debezium-server-iceberg 1.2.0.Final creates a namespace only when - # it is missing, so an existing dataset's storage URI places a new table. +def test_debezium_warning_says_the_server_refuses_to_boot(): + # memiiso/debezium-server-iceberg 1.2.0.Final IcebergConfig.java:44-45 + # declares debezium.sink.iceberg.warehouse as a String with no default, so + # the server does not start without one, whether or not the tables exist. binding = _binding("bigquery", "gcp", project="acme-proj") warning = _warehouse_warning(_dbz_contract(binding, {})) - assert "default storage location URI" in warning + assert "refuses to boot" in warning assert "debezium.sink.iceberg.warehouse" in warning + assert "default storage location URI" not in warning assert "createNamespace" not in warning From 38a6721fa5397796227a012188a73823ef08cefc Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:04:28 +0200 Subject: [PATCH 09/19] fix(iceberg): the plan path reads the bucket as written; a hand-written sink ignores it run_builds_from_args resolves a plan-embedded contract with resolve_contract_env_templates before dispatching, and that resolver renders an empty variable to "". The dispatcher then handed that partially resolved contract to contract_as_written, so with LAKE_ENV empty acme-{{ env.LAKE_ENV }}-lake reached the preflight as acme--lake and the sink was pushed s3://acme--lake/... . The dispatcher now takes the contract as written from its caller, and the binding match strips strings the way the plan resolver does. The preflight also refused a hand-written (non-deriving) sink config when a bucket variable it never reads was unset. Only a deriving build now waits on the bucket's variables. --- fluid_build/build_runners/base.py | 13 +- .../kafka_connect/iceberg_sink_validation.py | 25 +- .../test_fw2_catalog_table_plan_path.py | 300 ++++++++++++++++++ 3 files changed, 334 insertions(+), 4 deletions(-) create mode 100644 tests/build_runners/test_fw2_catalog_table_plan_path.py diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index 7b9f55c8..835bdab8 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -203,6 +203,7 @@ def _execute_acquisition_build( *, dry_run: bool, sample_rows: Any = None, + as_written: Optional[Dict[str, Any]] = None, ) -> int: """Dispatch an acquisition build to its runner. @@ -210,6 +211,10 @@ def _execute_acquisition_build( the registered callable. Capability negotiation runs first so a contract asking for an unsupported capability fails with the rich ``CapabilityMismatchError`` instead of running the wrong shape. + + ``as_written`` is ``contract`` before any ``{{ env.* }}`` resolution, when + the caller resolved some already (a plan-embedded contract); ``None`` + means ``contract`` is as written. """ engine = (build.get("engine") or "").strip().lower() @@ -252,7 +257,7 @@ def _execute_acquisition_build( # everything (including secret-shaped vars). Catalog-export paths use # ``cli/_common.py::resolve_contract_env_templates`` instead, which # leaves sensitive placeholders literal. - unresolved_contract = contract + unresolved_contract = contract if as_written is None else as_written build = _resolve_env_placeholders(build) contract = _resolve_env_placeholders(contract) @@ -1000,6 +1005,7 @@ def run_builds_from_args( 1, "contract_load_failed", {"path": str(contract_path), "error": str(exc)} ) contract = plan_data.get("contract") or {} + contract_as_written = contract if not contract: raise CLIError( 1, @@ -1072,6 +1078,7 @@ def run_builds_from_args( raise except Exception as e: raise CLIError(1, "contract_load_failed", {"path": str(contract_path), "error": str(e)}) + contract_as_written = contract if str(contract_path).lower().endswith((".tgz", ".tar.gz")): # A bundle's own directory is not where its contract lives: # anchor at the source contract its MANIFEST records. @@ -1162,12 +1169,16 @@ def run_builds_from_args( if is_acquisition_build(build): sample_rows = getattr(args, "sample_rows", None) + # A plan's contract had its ``{{ env.* }}`` templates resolved + # above, with an empty variable rendered to "": the Iceberg sink + # preflight reads the bucket template from the plan as written. result = _execute_acquisition_build( build, contract, contract_path.parent, dry_run=args.dry_run, sample_rows=sample_rows, + as_written=contract_as_written, ) _report_build(report, contract_path.parent, product_id, build_id, result, runs_before) if result == 0: diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 6000a0ec..0869ecd0 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -429,8 +429,12 @@ def _check_catalog( from ...providers._iceberg_catalog import unset_bucket_env_vars derived_warehouse = resolve_iceberg_catalog(binding, sink=sink, account_ref="").warehouse + # Only a deriving build reads the bucket; one that does not pushes the + # operator's map as written. as_written = _binding_as_written(binding) - waits_on = () if _overrides_warehouse(runtime) else unset_bucket_env_vars(as_written, kind) + waits_on: Tuple[str, ...] = () + if runtime.derives and not _overrides_warehouse(runtime): + waits_on = unset_bucket_env_vars(as_written, kind) if waits_on: required = "warehouse" in info.sink_requires issue = _unresolved_bucket_issue(bid, kind, as_written, waits_on, required=required) @@ -565,20 +569,35 @@ def _binding_as_written(binding: Mapping[str, Any]) -> Mapping[str, Any]: """``binding`` as the contract wrote it, while a runner runs; else ``binding``. The expose binding in the contract :func:`contract_as_written` holds whose - resolution by the dispatcher's own resolver is ``binding``. + resolution by the dispatcher's own resolver is ``binding``. Compared with + strings stripped: a plan-embedded contract is first resolved by + ``_contract_loader.resolve_contract_env_templates``, which strips every + string it renders. """ contract = _AS_WRITTEN.get() if contract is None or _AT_RUN.get() is None: return binding from ..base import _resolve_env_placeholders + target = _stripped(binding) for expose in contract.get("exposes") or []: raw = expose.get("binding") if isinstance(expose, Mapping) else None - if isinstance(raw, Mapping) and _resolve_env_placeholders(dict(raw)) == binding: + if isinstance(raw, Mapping) and _stripped(_resolve_env_placeholders(dict(raw))) == target: return raw return binding +def _stripped(node: Any) -> Any: + """``node`` with every string leaf stripped of surrounding whitespace.""" + if isinstance(node, str): + return node.strip() + if isinstance(node, Mapping): + return {k: _stripped(v) for k, v in node.items()} + if isinstance(node, (list, tuple)): + return [_stripped(v) for v in node] + return node + + def _unresolved_bucket_issue( bid: Any, kind: str, diff --git a/tests/build_runners/test_fw2_catalog_table_plan_path.py b/tests/build_runners/test_fw2_catalog_table_plan_path.py new file mode 100644 index 00000000..b3683985 --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_plan_path.py @@ -0,0 +1,300 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An env-templated bucket on the plan path, and a sink that never reads it. + +``fluid apply .json --mode amend-and-build`` runs +``run_builds_from_args(args, plan_data=...)``, which resolves the plan's +embedded contract with ``resolve_contract_env_templates`` before dispatching. +That resolver renders an empty variable to ``""``, so +``acme-{{ env.LAKE_ENV }}-lake`` reached the dispatcher as ``acme--lake`` and +the sink was pushed a warehouse in a bucket the contract does not name. + +A hand-written (non-deriving) sink config never reads the bucket, but the +preflight refused it when a bucket variable was unset. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +from fluid_build.build_runners import base +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) + +pytestmark = [pytest.mark.unit] + +_VAR = "FW2_PLAN_LAKE_ENV" +_PARTIAL = "acme-{{ env.%s }}-lake" % _VAR +_FULL = "{{ env.%s }}" % _VAR + + +def _contract(engine: str, catalog: str, bucket: str) -> Dict[str, Any]: + platform = "gcp" if catalog == "bigquery" else "aws" + loc: Dict[str, Any] = { + "database": "streaming", + "table": "orders", + "catalog": catalog, + "bucket": bucket, + } + if catalog == "bigquery": + loc["project"] = "acme-proj" + else: + loc["region"] = "eu-west-1" + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": {"platform": platform, "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + ], + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": ["orders"], + "properties": props, + } + ], + } + + +class _FakeConnect: + """Records the connector configs the Kafka Connect runner pushes.""" + + pushed: List[Tuple[str, Dict[str, Any]]] = [] + + def __init__(self, *_a: Any, **_k: Any) -> None: + pass + + def get_connector(self, _name: str) -> None: + return None + + def create_connector(self, name: str, cfg: Dict[str, Any]) -> None: + type(self).pushed.append((name, dict(cfg))) + + update_config = create_connector + + def __getattr__(self, _name: str) -> Any: + return lambda *a, **k: {"connector": {"state": "RUNNING"}, "tasks": [{"state": "RUNNING"}]} + + +@pytest.fixture +def fake_connect(monkeypatch): + from fluid_build.build_runners.kafka_connect import runner as kc_runner + + _FakeConnect.pushed = [] + monkeypatch.setattr(kc_runner, "KafkaConnectRestClient", _FakeConnect) + return _FakeConnect.pushed + + +def _set(monkeypatch, value: Optional[str]) -> None: + if value is None: + monkeypatch.delenv(_VAR, raising=False) + else: + monkeypatch.setenv(_VAR, value) + + +def _run_plan(contract: Dict[str, Any], tmp_path: Path, monkeypatch) -> int: + """``run_builds_from_args`` as ``fluid apply .json`` calls it.""" + monkeypatch.chdir(tmp_path) + plan_data = {"contract": contract} + plan = tmp_path / "plan.json" + plan.write_text(json.dumps(plan_data), encoding="utf-8") + args = argparse.Namespace( + contract=str(plan), + build_id=None, + dry_run=False, + env=None, + fail_fast=False, + sample_rows=None, + ) + return base.run_builds_from_args( + args, logging.getLogger("fw2.plan"), force_run=True, plan_data=plan_data + ) + + +def _log(caplog) -> str: + return "\n".join(r.getMessage() for r in caplog.records) + + +def _props_path(tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / "bronze.o" / "b1" / "application.properties" + + +# --------------------------------------------------------------------------- +# The plan path reads the bucket template as the plan wrote it +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc", "bigquery"]) +@pytest.mark.parametrize("bucket", [_PARTIAL, _FULL], ids=["partial", "full"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_plan_path_refuses_a_bucket_that_does_not_resolve( + fake_connect, tmp_path, monkeypatch, caplog, catalog, bucket, value +): + _set(monkeypatch, value) + contract = _contract("kafka-connect", catalog, bucket) + if catalog == "jdbc": + contract["exposes"][0]["binding"]["location"]["uri"] = "jdbc:postgresql://pg/ice" + with caplog.at_level(logging.INFO): + rc = _run_plan(contract, tmp_path, monkeypatch) + assert rc == 1 + # Nothing reached the cluster: no sink in a bucket the contract does not name. + assert fake_connect == [] + text = _log(caplog) + assert "iceberg sink preflight failed" in text + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert "acme--lake" not in text + + +def test_kc_plan_path_derives_from_the_resolved_bucket(fake_connect, tmp_path, monkeypatch): + _set(monkeypatch, "prod") + rc = _run_plan(_contract("kafka-connect", "dynamodb", _PARTIAL), tmp_path, monkeypatch) + assert rc == 0 + sinks = [cfg for name, cfg in fake_connect if name.endswith("-sink")] + assert len(sinks) == 1 + assert sinks[0]["iceberg.catalog.warehouse"] == "s3://acme-prod-lake/streaming/orders/" + + +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_debezium_plan_path_refuses_a_bucket_that_does_not_resolve( + tmp_path, monkeypatch, caplog, value +): + _set(monkeypatch, value) + with caplog.at_level(logging.INFO): + rc = _run_plan(_contract("debezium", "dynamodb", _PARTIAL), tmp_path, monkeypatch) + assert rc == 1 + assert not _props_path(tmp_path).exists() + text = _log(caplog) + assert "iceberg sink preflight failed" in text + assert f"{_VAR} is unset or empty in the runner's environment" in text + assert "acme--lake" not in text + + +def test_debezium_plan_path_derives_from_the_resolved_bucket(tmp_path, monkeypatch): + _set(monkeypatch, "prod") + # No server binary on PATH: the runner fails after writing the config. + _run_plan(_contract("debezium", "dynamodb", _PARTIAL), tmp_path, monkeypatch) + text = _props_path(tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://acme-prod-lake/streaming/orders/" in text + + +def test_plan_path_matches_a_binding_the_plan_resolver_stripped( + fake_connect, tmp_path, monkeypatch, caplog +): + # ``resolve_contract_env_templates`` strips every string it renders; the + # binding is still found in the plan as written. + _set(monkeypatch, "") + contract = _contract("kafka-connect", "dynamodb", " %s " % _PARTIAL) + with caplog.at_level(logging.INFO): + assert _run_plan(contract, tmp_path, monkeypatch) == 1 + assert fake_connect == [] + assert f"{_VAR} is unset or empty in the runner's environment" in _log(caplog) + + +# --------------------------------------------------------------------------- +# A hand-written sink config never reads the bucket +# --------------------------------------------------------------------------- + + +_HANDWRITTEN_BQ = { + "connector.class": "org.apache.iceberg.connect.IcebergSinkConnector", + "topics": "orders", + "iceberg.tables": "streaming.orders", + "iceberg.catalog.type": "bigquery", + "iceberg.catalog.gcp.bigquery.project-id": "acme-proj", +} + + +def _handwritten_kc(catalog: str = "bigquery") -> Dict[str, Any]: + contract = _contract("kafka-connect", catalog, _FULL) + cfg = dict(_HANDWRITTEN_BQ) + if catalog == "dynamodb": + cfg.pop("iceberg.catalog.type") + cfg.pop("iceberg.catalog.gcp.bigquery.project-id") + cfg["iceberg.catalog.catalog-impl"] = "org.apache.iceberg.aws.dynamodb.DynamoDbCatalog" + cfg["iceberg.catalog.warehouse"] = "s3://ops/wh/" + contract["builds"][0]["properties"]["kafka-connect"]["sink_connector_config"] = cfg + return contract + + +@pytest.mark.parametrize("catalog", ["bigquery", "dynamodb"]) +def test_handwritten_kc_config_ignores_an_unset_bucket_variable( + fake_connect, tmp_path, monkeypatch, catalog +): + _set(monkeypatch, None) + contract = _handwritten_kc(catalog) + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert not [w for w in warnings if _VAR in w], warnings + assert iceberg_sink_preflight(contract, "b1") is None + + monkeypatch.chdir(tmp_path) + rc = base._execute_acquisition_build(contract["builds"][0], contract, tmp_path, dry_run=False) + assert rc == 0 + sinks = [cfg for name, cfg in fake_connect if name.endswith("-sink")] + assert len(sinks) == 1 + pushed = {k: v for k, v in sinks[0].items() if k.startswith("iceberg.catalog.")} + written = contract["builds"][0]["properties"]["kafka-connect"]["sink_connector_config"] + assert pushed == {k: v for k, v in written.items() if k.startswith("iceberg.catalog.")} + + +def test_handwritten_debezium_config_ignores_an_unset_bucket_variable(monkeypatch): + _set(monkeypatch, None) + contract = _contract("debezium", "dynamodb", _FULL) + contract["builds"][0]["properties"]["debezium"]["server"]["sink"] = { + "config": { + "catalog-impl": "org.apache.iceberg.aws.dynamodb.DynamoDbCatalog", + "warehouse": "s3://ops/wh/", + } + } + errors, warnings = validate_iceberg_sink(contract) + assert errors == [] + assert not [w for w in warnings if _VAR in w], warnings + assert iceberg_sink_preflight(contract, "b1") is None + + +def test_a_deriving_build_still_waits_on_the_bucket_variable(monkeypatch): + # Control for the tests above: the same contract, derived, is named. + _set(monkeypatch, None) + contract = _contract("kafka-connect", "bigquery", _FULL) + warnings = validate_iceberg_sink(contract)[1] + assert [w for w in warnings if _VAR in w], warnings From 50477201061443a746524a77516c1e6b3a08b13c Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:23:10 +0200 Subject: [PATCH 10/19] fix(iceberg): a blank warehouse counts as unset; a BigQuery sink without auto-create only warns A whitespace-only location.warehouse reached the sink verbatim. It now counts as unset, so a DynamoDB or JDBC sink derives the warehouse from the bucket or is refused. A Kafka Connect BigQuery sink reads the warehouse only to auto-create tables (iceberg.tables.auto-create-enabled defaults to false, apache-iceberg-1.10.0 IcebergSinkConfig.java:154-159). When its bucket variable is unset or empty, the run now refuses only with auto-create on; otherwise it warns and pushes no warehouse rather than one in a bucket the contract does not name. --- .../kafka_connect/iceberg_sink_validation.py | 71 ++++++++++++-- .../build_runners/kafka_connect/runner.py | 7 ++ fluid_build/providers/_iceberg_catalog.py | 2 +- .../test_fw2_catalog_table_dispatch.py | 40 ++++++-- .../test_fw2_catalog_table_plan_path.py | 9 +- .../test_fw2_catalog_table_whitespace.py | 98 +++++++++++++++++++ 6 files changed, 209 insertions(+), 18 deletions(-) create mode 100644 tests/build_runners/test_fw2_catalog_table_whitespace.py diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 0869ecd0..282ed5ea 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -350,7 +350,15 @@ def _check_build( "streamingSink.routeField" ) - _check_catalog(bid, binding, sink, runtime, errors, warnings) + _check_catalog( + bid, + binding, + sink, + runtime, + errors, + warnings, + auto_creates=_auto_creates(streaming, runtime), + ) def _check_catalog( @@ -360,6 +368,8 @@ def _check_catalog( runtime: IcebergSinkPlan, errors: List[str], warnings: List[str], + *, + auto_creates: bool = False, ) -> None: """The catalog checks (4-6), every one read off the kind's table row.""" loc = binding.get("location") or {} @@ -422,9 +432,11 @@ def _check_catalog( # into the config it pushes. A bucket whose ``{{ env.* }}`` template # does not resolve here is explicit, so ``fluid validate`` only warns: # the runner's preflight, in the environment the sink config is - # derived in, refuses it (BigQuery's too, which needs no warehouse to - # start: the alternative is a warehouse in a bucket the contract does - # not name). Glue requires nothing (the warehouse falls back), so its + # derived in, refuses it when the sink would use that warehouse: a + # DynamoDB / JDBC catalog needs one to start, Debezium Server needs one + # to boot, and a Kafka Connect BigQuery sink reads it only to + # auto-create tables (see the BigQuery warning below). Otherwise the run + # warns too. Glue requires nothing (the warehouse falls back), so its # region stays advisory. from ...providers._iceberg_catalog import unset_bucket_env_vars @@ -437,9 +449,12 @@ def _check_catalog( waits_on = unset_bucket_env_vars(as_written, kind) if waits_on: required = "warehouse" in info.sink_requires - issue = _unresolved_bucket_issue(bid, kind, as_written, waits_on, required=required) + refused = required or runtime.engine == "debezium" or auto_creates + issue = _unresolved_bucket_issue( + bid, kind, as_written, waits_on, required=required, refused=refused + ) run_only = _AT_RUN.get() - if run_only is None: + if run_only is None or not refused: warnings.append(issue) else: errors.append(issue) @@ -577,6 +592,25 @@ def _binding_as_written(binding: Mapping[str, Any]) -> Mapping[str, Any]: contract = _AS_WRITTEN.get() if contract is None or _AT_RUN.get() is None: return binding + return _match_as_written(contract, binding) + + +def unresolved_bucket_vars_at_run(binding: Mapping[str, Any], kind: str) -> Tuple[str, ...]: + """The unset or empty variables in ``binding``'s bucket as the contract wrote it. + + For a runner deriving inside :func:`contract_as_written`, where ``binding`` + is already resolved; ``()`` outside that scope. + """ + contract = _AS_WRITTEN.get() + if contract is None: + return () + from ...providers._iceberg_catalog import unset_bucket_env_vars + + return unset_bucket_env_vars(_match_as_written(contract, binding), kind) + + +def _match_as_written(contract: Mapping[str, Any], binding: Mapping[str, Any]) -> Mapping[str, Any]: + """The expose binding in ``contract`` that resolves to ``binding``, else ``binding``.""" from ..base import _resolve_env_placeholders target = _stripped(binding) @@ -605,6 +639,7 @@ def _unresolved_bucket_issue( names: Tuple[str, ...], *, required: bool, + refused: bool, ) -> str: """Check 4's message for a warehouse that waits on unset or empty bucket variables.""" raw = (binding.get("location") or {}).get("bucket") @@ -622,12 +657,30 @@ def _unresolved_bucket_issue( return ( f"iceberg sink (build {bid!r}): the {kind} catalog's warehouse derives from " f"binding.location.bucket {raw!r}, and {unset} here, so it cannot be derived at " - f"validate time. Set {'it' if one else 'them'} where the sink runs: the runner " - "refuses the build when the bucket does not resolve there" - + (f", because {refuses}" if required else "") + f"validate time. Set {'it' if one else 'them'} where the sink runs" + + ( + ": the runner refuses the build when the bucket does not resolve there" + + (f", because {refuses}" if required else "") + if refused + else "" + ) ) +def _auto_creates(streaming: Mapping[str, Any], runtime: IcebergSinkPlan) -> bool: + """Does this Kafka Connect sink auto-create tables? Its default is off. + + ``streamingSink.autoCreate`` sets ``iceberg.tables.auto-create-enabled``, + and an override map merged after it wins (apache-iceberg-1.10.0 + IcebergSinkConfig.java:154-159 defaults the key to false). + """ + value: Any = streaming.get("autoCreate") + for _label, cfg in runtime.overrides: + if "iceberg.tables.auto-create-enabled" in cfg: + value = cfg["iceberg.tables.auto-create-enabled"] + return str(value).strip().lower() == "true" + + def _requires_hint(key: str, kind: str, info: CatalogKind) -> str: """What a missing ``sink_requires`` key is to the sink, for check 4's error.""" if key == "warehouse" and info.family == FAMILY_REST: diff --git a/fluid_build/build_runners/kafka_connect/runner.py b/fluid_build/build_runners/kafka_connect/runner.py index 29067489..e2eb63fa 100644 --- a/fluid_build/build_runners/kafka_connect/runner.py +++ b/fluid_build/build_runners/kafka_connect/runner.py @@ -392,6 +392,13 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: schema_registry_url=sr_url, delivery_guarantee=(props.get("delivery") or {}).get("guarantee"), ) + # A bucket whose variable is unset or empty here resolved to one the + # contract does not name. The preflight lets that through only for a + # sink that never reads the warehouse (BigQuery without auto-create). + from .iceberg_sink_validation import unresolved_bucket_vars_at_run + + if unresolved_bucket_vars_at_run(binding, resolved.kind): + derived.pop("iceberg.catalog.warehouse", None) sink_config = {**derived, **(sink_config or {})} connector_name = kc_props.get("connector_name") or f"forge-{ctx.product_id.replace('.', '-')}" diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 4898b104..fc542404 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -452,7 +452,7 @@ def resolve_iceberg_catalog( # unknown kind keeps the historic REST fallback, and ``fluid validate`` # refuses it). The FileIO follows the WAREHOUSE scheme so GCS (gs://) and # ADLS (abfss://) work, not just S3 (RFC §6.3 — PR7's REST + GCP profiles). - warehouse = loc.get("warehouse") or "" + warehouse = str(loc.get("warehouse") or "").strip() region = loc.get("region") extra: Dict[str, str] = {} if kind == "bigquery": diff --git a/tests/build_runners/test_fw2_catalog_table_dispatch.py b/tests/build_runners/test_fw2_catalog_table_dispatch.py index 1df1bcc6..d02bd156 100644 --- a/tests/build_runners/test_fw2_catalog_table_dispatch.py +++ b/tests/build_runners/test_fw2_catalog_table_dispatch.py @@ -59,7 +59,7 @@ def _binding(catalog: str, bucket: str, **location: Any) -> Dict[str, Any]: return {"platform": platform, "format": "iceberg", "location": loc} -def _contract(binding: Dict[str, Any], engine: str) -> Dict[str, Any]: +def _contract(binding: Dict[str, Any], engine: str, *, auto_create: bool = False) -> Dict[str, Any]: props: Dict[str, Any] = { "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, "sink": {"format": "iceberg"}, @@ -70,6 +70,8 @@ def _contract(binding: Dict[str, Any], engine: str) -> Dict[str, Any]: props["kafka-connect"] = { "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} } + if auto_create: + props["kafka-connect"]["streamingSink"] = {"autoCreate": True} return { "fluidVersion": "0.7.6", "kind": "DataProduct", @@ -167,7 +169,10 @@ def test_kc_dispatch_refuses_a_bucket_that_does_not_resolve( ): _set(monkeypatch, value) extra = {"uri": "jdbc:postgresql://pg:5432/iceberg"} if catalog == "jdbc" else {} - contract = _contract(_binding(catalog, bucket, **extra), "kafka-connect") + # A Kafka Connect BigQuery sink reads the warehouse only to auto-create + # tables, so it is refused only with auto-create on. + binding = _binding(catalog, bucket, **extra) + contract = _contract(binding, "kafka-connect", auto_create=catalog == "bigquery") with caplog.at_level(logging.INFO): rc = _dispatch(contract, tmp_path, monkeypatch) assert rc == 1 @@ -258,7 +263,8 @@ def test_debezium_dispatch_derives_from_the_resolved_bucket(tmp_path, monkeypatc [("mylake", "gs://mylake"), ("", ""), (None, "")], ids=["set", "empty", "unset"], ) -def test_bigquery_env_bucket_derive_and_validate(monkeypatch, value, warehouse): +@pytest.mark.parametrize("auto_create", [False, True], ids=["no-autocreate", "autocreate"]) +def test_bigquery_env_bucket_derive_and_validate(monkeypatch, value, warehouse, auto_create): _set(monkeypatch, value) binding = _binding("bigquery", _FULL) resolved = resolve_iceberg_catalog(binding) @@ -266,7 +272,7 @@ def test_bigquery_env_bucket_derive_and_validate(monkeypatch, value, warehouse): assert "{{" not in resolved.warehouse assert resolved.extra_catalog_props["gcp.bigquery.project-id"] == "acme-proj" - contract = _contract(binding, "kafka-connect") + contract = _contract(binding, "kafka-connect", auto_create=auto_create) errors, warnings = validate_iceberg_sink(contract) assert errors == [] named = [w for w in warnings if _VAR in w] @@ -279,8 +285,30 @@ def test_bigquery_env_bucket_derive_and_validate(monkeypatch, value, warehouse): assert len(named) == 1, warnings assert f"{_VAR} is unset or empty here" in named[0] assert not [w for w in warnings if "no gs:// warehouse" in w], warnings - assert refusal is not None - assert f"{_VAR} is unset or empty in the runner's environment" in refusal + # The run refuses only when the sink would read the warehouse. + assert ("the runner refuses the build" in named[0]) is auto_create + if auto_create: + assert refusal is not None + assert f"{_VAR} is unset or empty in the runner's environment" in refusal + else: + assert refusal is None + + +@pytest.mark.parametrize("bucket", [_FULL, _PARTIAL], ids=["full", "partial"]) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_bigquery_without_autocreate_runs_and_names_no_other_bucket( + fake_connect, tmp_path, monkeypatch, caplog, bucket, value +): + _set(monkeypatch, value) + contract = _contract(_binding("bigquery", bucket), "kafka-connect") + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 0, caplog.text + sinks = [cfg for _name, cfg in fake_connect if cfg.get("iceberg.catalog.type") == "bigquery"] + assert len(sinks) == 1, fake_connect + # No warehouse in a bucket the contract does not name (acme--lake, gs://). + pushed = sinks[0].get("iceberg.catalog.warehouse", "") + assert pushed in ("",), sinks[0] def test_bigquery_partial_bucket_with_an_empty_variable_derives_none(monkeypatch): diff --git a/tests/build_runners/test_fw2_catalog_table_plan_path.py b/tests/build_runners/test_fw2_catalog_table_plan_path.py index b3683985..f371ebe6 100644 --- a/tests/build_runners/test_fw2_catalog_table_plan_path.py +++ b/tests/build_runners/test_fw2_catalog_table_plan_path.py @@ -48,7 +48,9 @@ _FULL = "{{ env.%s }}" % _VAR -def _contract(engine: str, catalog: str, bucket: str) -> Dict[str, Any]: +def _contract( + engine: str, catalog: str, bucket: str, *, auto_create: bool = False +) -> Dict[str, Any]: platform = "gcp" if catalog == "bigquery" else "aws" loc: Dict[str, Any] = { "database": "streaming", @@ -70,6 +72,8 @@ def _contract(engine: str, catalog: str, bucket: str) -> Dict[str, Any]: props["kafka-connect"] = { "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} } + if auto_create: + props["kafka-connect"]["streamingSink"] = {"autoCreate": True} return { "fluidVersion": "0.7.6", "kind": "DataProduct", @@ -171,7 +175,8 @@ def test_kc_plan_path_refuses_a_bucket_that_does_not_resolve( fake_connect, tmp_path, monkeypatch, caplog, catalog, bucket, value ): _set(monkeypatch, value) - contract = _contract("kafka-connect", catalog, bucket) + # A Kafka Connect BigQuery sink is refused only with auto-create on. + contract = _contract("kafka-connect", catalog, bucket, auto_create=catalog == "bigquery") if catalog == "jdbc": contract["exposes"][0]["binding"]["location"]["uri"] = "jdbc:postgresql://pg/ice" with caplog.at_level(logging.INFO): diff --git a/tests/build_runners/test_fw2_catalog_table_whitespace.py b/tests/build_runners/test_fw2_catalog_table_whitespace.py new file mode 100644 index 00000000..949417ce --- /dev/null +++ b/tests/build_runners/test_fw2_catalog_table_whitespace.py @@ -0,0 +1,98 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A whitespace-only ``location.warehouse`` counts as unset. + +``' '`` used to reach the sink as its warehouse. Now a DynamoDB or JDBC sink +derives the warehouse from the bucket, or validate and the preflight refuse. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + + +def _binding(catalog: str, **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = {"database": "streaming", "table": "orders", "catalog": catalog} + loc["region"] = "eu-west-1" + if catalog == "jdbc": + loc["uri"] = "jdbc:postgresql://pg:5432/iceberg" + loc.update(location) + return {"platform": "aws", "format": "iceberg", "location": loc} + + +def _contract(binding: Dict[str, Any], engine: str) -> Dict[str, Any]: + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": binding, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + ], + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": ["orders"], + "properties": props, + } + ], + } + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("blank", [" ", "\t", ""], ids=["spaces", "tab", "empty"]) +def test_a_blank_warehouse_falls_back_to_the_bucket(catalog, blank): + resolved = resolve_iceberg_catalog(_binding(catalog, bucket="lake", warehouse=blank)) + assert resolved.warehouse == "s3://lake/streaming/orders/" + dbz = emit_debezium_iceberg_sink_config(resolved) + assert dbz.get("warehouse") == "s3://lake/streaming/orders/", dbz + + +@pytest.mark.parametrize("catalog", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_blank_warehouse_and_no_bucket_is_refused(catalog, engine): + contract = _contract(_binding(catalog, warehouse=" "), engine) + errors, _warnings = validate_iceberg_sink(contract) + assert any(f"{catalog} catalog requires binding.location.warehouse" in e for e in errors) + assert iceberg_sink_preflight(contract, "b1") is not None From 91fd3973eaeeb4f1edd77af874a6d4df98bf1709 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:46:58 +0200 Subject: [PATCH 11/19] fix(iceberg): keep an override warehouse; a blank BigQuery project counts as missing 50477201 dropped the derived warehouse when the bucket variable was unset, but iceberg_catalog_overrides is already merged into the derived config, so an operator's override warehouse was dropped too and a DynamoDB, JDBC or BigQuery sink was pushed without one. The runner now drops only a warehouse check 4 would have derived: none for a build that does not derive, or whose overrides set the warehouse. A whitespace-only location.project passed as gcp.bigquery.project-id. The project and region are stripped, and check 4 tests the stripped value, so a blank project is the missing-project error. --- .../kafka_connect/iceberg_sink_validation.py | 14 ++++--- .../build_runners/kafka_connect/runner.py | 5 ++- fluid_build/providers/_iceberg_catalog.py | 10 +++-- .../test_fw2_catalog_table_dispatch.py | 37 +++++++++++++++++- .../test_fw2_catalog_table_whitespace.py | 38 +++++++++++++++++++ 5 files changed, 92 insertions(+), 12 deletions(-) diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index 282ed5ea..602a037e 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -462,7 +462,7 @@ def _check_catalog( for key in info.sink_requires: if key == "warehouse" and waits_on: continue # reported above, naming the variables - present = derived_warehouse if key == "warehouse" else loc.get(key) + present = derived_warehouse if key == "warehouse" else str(loc.get(key) or "").strip() start_prop = _start_only_property(key, kind) if not present and start_prop is not None: present = _overrides_catalog_property(runtime, start_prop) @@ -595,14 +595,18 @@ def _binding_as_written(binding: Mapping[str, Any]) -> Mapping[str, Any]: return _match_as_written(contract, binding) -def unresolved_bucket_vars_at_run(binding: Mapping[str, Any], kind: str) -> Tuple[str, ...]: - """The unset or empty variables in ``binding``'s bucket as the contract wrote it. +def unresolved_bucket_vars_at_run( + binding: Mapping[str, Any], kind: str, runtime: IcebergSinkPlan +) -> Tuple[str, ...]: + """The unset or empty variables the derived warehouse waits on, as check 4 reads them. For a runner deriving inside :func:`contract_as_written`, where ``binding`` - is already resolved; ``()`` outside that scope. + is already resolved: the variables in its bucket as the contract wrote it. + ``()`` outside that scope, for a build that does not derive, and when an + override map sets the warehouse (the override is what reaches the sink). """ contract = _AS_WRITTEN.get() - if contract is None: + if contract is None or not runtime.derives or _overrides_warehouse(runtime): return () from ...providers._iceberg_catalog import unset_bucket_env_vars diff --git a/fluid_build/build_runners/kafka_connect/runner.py b/fluid_build/build_runners/kafka_connect/runner.py index e2eb63fa..d65cb85c 100644 --- a/fluid_build/build_runners/kafka_connect/runner.py +++ b/fluid_build/build_runners/kafka_connect/runner.py @@ -394,10 +394,11 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: ) # A bucket whose variable is unset or empty here resolved to one the # contract does not name. The preflight lets that through only for a - # sink that never reads the warehouse (BigQuery without auto-create). + # sink that never reads the warehouse (BigQuery without auto-create); + # an override warehouse is the operator's and stays. from .iceberg_sink_validation import unresolved_bucket_vars_at_run - if unresolved_bucket_vars_at_run(binding, resolved.kind): + if unresolved_bucket_vars_at_run(binding, resolved.kind, sink_plan): derived.pop("iceberg.catalog.warehouse", None) sink_config = {**derived, **(sink_config or {})} diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index fc542404..90d954cb 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -463,10 +463,12 @@ def resolve_iceberg_catalog( # location key instead. rendered = {**loc, "bucket": _rendered_bucket(loc)} warehouse = iceberg_storage_uri({"location": rendered}, scheme="gs") - if loc.get("project"): - extra[BIGQUERY_PROJECT_ID] = str(loc["project"]) - if region: - extra[BIGQUERY_LOCATION] = str(region) + project = str(loc.get("project") or "").strip() + if project: + extra[BIGQUERY_PROJECT_ID] = project + location = str(region or "").strip() + if location: + extra[BIGQUERY_LOCATION] = location region = None elif not warehouse and kind in BUCKET_WAREHOUSE_KINDS: warehouse = _bucket_warehouse(binding) diff --git a/tests/build_runners/test_fw2_catalog_table_dispatch.py b/tests/build_runners/test_fw2_catalog_table_dispatch.py index d02bd156..0910d96e 100644 --- a/tests/build_runners/test_fw2_catalog_table_dispatch.py +++ b/tests/build_runners/test_fw2_catalog_table_dispatch.py @@ -59,7 +59,13 @@ def _binding(catalog: str, bucket: str, **location: Any) -> Dict[str, Any]: return {"platform": platform, "format": "iceberg", "location": loc} -def _contract(binding: Dict[str, Any], engine: str, *, auto_create: bool = False) -> Dict[str, Any]: +def _contract( + binding: Dict[str, Any], + engine: str, + *, + auto_create: bool = False, + overrides: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: props: Dict[str, Any] = { "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, "sink": {"format": "iceberg"}, @@ -72,6 +78,8 @@ def _contract(binding: Dict[str, Any], engine: str, *, auto_create: bool = False } if auto_create: props["kafka-connect"]["streamingSink"] = {"autoCreate": True} + if overrides: + props["kafka-connect"]["iceberg_catalog_overrides"] = dict(overrides) return { "fluidVersion": "0.7.6", "kind": "DataProduct", @@ -358,3 +366,30 @@ def test_kafka_connect_bigquery_no_warehouse_warning_is_unchanged(): warnings = validate_iceberg_sink(_contract(binding, "kafka-connect"))[1] hit = [w for w in warnings if "no gs:// warehouse" in w] assert len(hit) == 1 and "without auto-create, tables that exist need none" in hit[0] + + +@pytest.mark.parametrize( + "catalog, warehouse", + [ + ("dynamodb", "s3://ops-lake/wh"), + ("jdbc", "s3://ops-lake/wh"), + ("bigquery", "gs://ops-lake/wh"), + ], +) +@pytest.mark.parametrize("value", [None, ""], ids=["unset", "empty"]) +def test_kc_dispatch_keeps_an_override_warehouse( + fake_connect, tmp_path, monkeypatch, caplog, catalog, warehouse, value +): + _set(monkeypatch, value) + extra = {"uri": "jdbc:postgresql://pg:5432/iceberg"} if catalog == "jdbc" else {} + contract = _contract( + _binding(catalog, _FULL, **extra), + "kafka-connect", + overrides={"iceberg.catalog.warehouse": warehouse}, + ) + with caplog.at_level(logging.INFO): + rc = _dispatch(contract, tmp_path, monkeypatch) + assert rc == 0, caplog.text + sinks = [cfg for _name, cfg in fake_connect if "iceberg.tables" in cfg] + assert len(sinks) == 1, fake_connect + assert sinks[0].get("iceberg.catalog.warehouse") == warehouse, sinks[0] diff --git a/tests/build_runners/test_fw2_catalog_table_whitespace.py b/tests/build_runners/test_fw2_catalog_table_whitespace.py index 949417ce..8880b1a4 100644 --- a/tests/build_runners/test_fw2_catalog_table_whitespace.py +++ b/tests/build_runners/test_fw2_catalog_table_whitespace.py @@ -96,3 +96,41 @@ def test_a_blank_warehouse_and_no_bucket_is_refused(catalog, engine): errors, _warnings = validate_iceberg_sink(contract) assert any(f"{catalog} catalog requires binding.location.warehouse" in e for e in errors) assert iceberg_sink_preflight(contract, "b1") is not None + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_blank_bigquery_project_is_refused(engine): + binding = { + "platform": "gcp", + "format": "iceberg", + "location": { + "database": "streaming", + "table": "orders", + "catalog": "bigquery", + "bucket": "lake", + "project": " ", + }, + } + assert "gcp.bigquery.project-id" not in resolve_iceberg_catalog(binding).extra_catalog_props + contract = _contract(binding, engine) + errors, _warnings = validate_iceberg_sink(contract) + assert any("bigquery catalog requires binding.location.project" in e for e in errors), errors + assert iceberg_sink_preflight(contract, "b1") is not None + + +def test_a_padded_bigquery_project_and_region_are_stripped(): + binding = { + "platform": "gcp", + "format": "iceberg", + "location": { + "database": "streaming", + "table": "orders", + "catalog": "bigquery", + "bucket": "lake", + "project": " acme-proj ", + "region": " EU ", + }, + } + props = resolve_iceberg_catalog(binding).extra_catalog_props + assert props["gcp.bigquery.project-id"] == "acme-proj" + assert props["gcp.bigquery.location"] == "EU" From 8e6b7e9a47d4b355d4cee9a0f5534a6b6619333c Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:54:23 +0200 Subject: [PATCH 12/19] fix(iceberg): a Debezium sink over bucket-derived DynamoDB/JDBC exposes of one database is one catalog A DynamoDB or JDBC warehouse derived from location.bucket is a per-table prefix, as Glue's is, so two such exposes of one database no longer read as two catalogs to the embedded Debezium join: only their bucket is compared. A BigQuery project or location that differs is named as the binding's project or region, not as catalog properties. Adds tests for the expose resolver and the catalog derivation together. --- fluid_build/build_runners/debezium/runner.py | 5 +- fluid_build/providers/_iceberg_catalog.py | 48 ++- .../test_fw2_integration_seams.py | 364 ++++++++++++++++++ 3 files changed, 409 insertions(+), 8 deletions(-) create mode 100644 tests/build_runners/test_fw2_integration_seams.py diff --git a/fluid_build/build_runners/debezium/runner.py b/fluid_build/build_runners/debezium/runner.py index 3d96e6e8..aafbce7e 100644 --- a/fluid_build/build_runners/debezium/runner.py +++ b/fluid_build/build_runners/debezium/runner.py @@ -553,8 +553,9 @@ def _execute_debezium_server( # The exposes the build's outputs name, as `fluid validate` resolves # them. They share one database and one catalog, so the first one's - # binding gives the namespace and the catalog block (for Glue, whose - # warehouse is a per-table prefix, the first one's prefix). + # binding gives the namespace and the catalog block (for Glue, and for + # a DynamoDB / JDBC warehouse derived from the bucket, whose warehouse + # is a per-table prefix, the first one's prefix). targets, _error = iceberg_sink_target(ctx.contract, build, sink_plan) if targets: resolved = resolve_iceberg_catalog( diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 1e0d34b8..31032c5f 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -457,6 +457,11 @@ def resolve_iceberg_sink_exposes( } ) +#: The binding setting each catalog-specific property is resolved from. +_CATALOG_PROP_SOURCES: Mapping[str, str] = MappingProxyType( + {BIGQUERY_PROJECT_ID: "project", BIGQUERY_LOCATION: "region"} +) + def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: """The catalog settings on which ``exposes`` resolve differently, or ``[]``. @@ -464,11 +469,13 @@ def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: Compares every :class:`ResolvedIcebergCatalog` field except the table identity and the per-table write defaults. A Glue warehouse is per table too (``get_iceberg_warehouse`` derives ``s3://{bucket}/{database}/{table}/``), - while a REST catalog's warehouse is the catalog's name. + while a REST catalog's warehouse is the catalog's name. A DynamoDB or JDBC + warehouse derived from ``location.bucket`` (:func:`_bucket_warehouse`) is + a per-table prefix the same way, so only its bucket is compared. """ from dataclasses import fields as dataclass_fields - resolved = [resolve_iceberg_catalog(e.get("binding") or {}) for e in exposes] + resolved = [_bucket_rooted(e.get("binding") or {}) for e in exposes] if len(resolved) < 2: return [] per_table = {"fq_table", "id_columns", "partition_by"} @@ -478,13 +485,42 @@ def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: for f in dataclass_fields(ResolvedIcebergCatalog): if f.name in per_table: continue - if any(getattr(r, f.name) != getattr(resolved[0], f.name) for r in resolved[1:]): - source = _CATALOG_FIELD_SOURCES.get(f.name, f.name) - if source not in differs: - differs.append(source) + if f.name == "extra_catalog_props": + # Named by the binding setting each property comes from. + props = sorted({k for r in resolved for k in r.extra_catalog_props}) + sources = [ + _CATALOG_PROP_SOURCES.get(k, "catalog properties") + for k in props + if any( + r.extra_catalog_props.get(k) != resolved[0].extra_catalog_props.get(k) + for r in resolved[1:] + ) + ] + elif any(getattr(r, f.name) != getattr(resolved[0], f.name) for r in resolved[1:]): + sources = [_CATALOG_FIELD_SOURCES.get(f.name, f.name)] + else: + sources = [] + differs.extend(s for s in dict.fromkeys(sources) if s not in differs) return differs +def _bucket_rooted(binding: Mapping[str, Any]) -> ResolvedIcebergCatalog: + """:func:`resolve_iceberg_catalog` of ``binding``, a bucket-derived warehouse cut to its bucket. + + ``s3://lake/sales/orders/`` becomes ``s3://lake``: the path + :func:`_bucket_warehouse` appends defaults to the table's own prefix. A + ``location.warehouse`` the binding sets is kept whole. + """ + from dataclasses import replace + + resolved = resolve_iceberg_catalog(binding) + explicit = str((binding.get("location") or {}).get("warehouse") or "").strip() + if resolved.kind not in BUCKET_WAREHOUSE_KINDS or explicit or "://" not in resolved.warehouse: + return resolved + scheme, rest = resolved.warehouse.split("://", 1) + return replace(resolved, warehouse=f"{scheme}://{rest.split('/', 1)[0]}") + + #: Object-store URI schemes, and the Iceberg ``FileIO`` each one needs. THE one #: scheme list: the resolver picks a FileIO from it and the sink validator #: refuses a name-only warehouse that carries one. diff --git a/tests/build_runners/test_fw2_integration_seams.py b/tests/build_runners/test_fw2_integration_seams.py new file mode 100644 index 00000000..fe9511a7 --- /dev/null +++ b/tests/build_runners/test_fw2_integration_seams.py @@ -0,0 +1,364 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The sink-expose resolver and the catalog-kind derivation, composed. + +The resolver picks the exposes a derived streaming sink writes from the +build's ``outputs``; the catalog table derives what each kind needs to start +(a DynamoDB / JDBC warehouse from the bucket, the BigQuery project and +location). Each was tested on its own. Here a build writes a DynamoDB expose +and another a BigQuery expose in one contract, an embedded Debezium build +writes two exposes of one JDBC or DynamoDB database, and a Confluent Tableflow +expose with no catalog sits beside a self-managed sink with access grants. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest +import yaml + +from fluid_build.api.runner import RunState +from fluid_build.build_runners._acquisition_common import build_acquisition_run_context +from fluid_build.build_runners.debezium.runner import DebeziumRunner +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.build_runners.kafka_connect.runner import execute_kafka_connect_build +from fluid_build.cli import policy_compile +from fluid_build.policy.compiler import compile_policy + +pytestmark = [pytest.mark.unit] + +_SOURCE = { + "kind": "postgres", + "connection": {"host": "db", "port": 5432, "database": "app", "user": "u", "password": "p"}, + "mode": "incremental_append", + "watermark": {"strategy": "high_water_mark", "allowedLateness": "PT5M"}, +} + +_JDBC_URI = "jdbc:postgresql://catalog-db:5432/iceberg" + + +def _expose(eid: str, platform: str, **location: Any) -> Dict[str, Any]: + return { + "exposeId": eid, + "kind": "table", + "binding": {"platform": platform, "format": "iceberg", "location": location}, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + + +def _kc_build(bid: str, outputs: Optional[List[str]], stream: str) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": bid, + "pattern": "acquisition", + "engine": "kafka-connect", + "capabilities": ["streaming"], + "properties": { + "source": {**copy.deepcopy(_SOURCE), "streams": [stream]}, + "sink": {"format": "iceberg"}, + "kafka-connect": { + "deployment": {"server_url": "http://kafka-connect.test:8083"}, + "connector_name": f"src-{bid}", + }, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _dbz_build(outputs: Optional[List[str]]) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": "ingest", + "pattern": "acquisition", + "engine": "debezium", + "capabilities": ["cdc", "streaming"], + "properties": { + "source": { + **copy.deepcopy(_SOURCE), + "mode": "cdc", + "streams": ["public.orders", "public.refunds"], + }, + "sink": {"format": "iceberg"}, + "debezium": {"deployment": {"mode": "embedded"}, "server": {"sink": {}}}, + }, + } + if outputs is not None: + build["outputs"] = outputs + return build + + +def _contract( + exposes: List[Dict[str, Any]], + builds: List[Dict[str, Any]], + *, + grants: bool = False, +) -> Dict[str, Any]: + doc: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.cdc", + "name": "Sales CDC", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "x@y.z"}}, + "exposes": copy.deepcopy(exposes), + "builds": copy.deepcopy(builds), + } + if grants: + doc["accessPolicy"] = { + "grants": [{"principal": "group:analysts@example.com", "permissions": ["read"]}] + } + return doc + + +# ── a DynamoDB expose and a BigQuery expose, one Kafka Connect build each ──── + +DDB = _expose( + "orders", + "aws", + database="sales", + table="orders", + bucket="lake", + region="us-east-1", + catalog="dynamodb", +) +BQ = _expose( + "events", + "gcp", + database="analytics", + table="events", + bucket="gbkt", + region="europe-west1", + project="acme-proj", + catalog="bigquery", +) + + +def _ddb_and_bq() -> Dict[str, Any]: + return _contract( + [DDB, BQ], + [ + _kc_build("ingest_orders", ["orders"], "public.orders"), + _kc_build("ingest_events", ["events"], "public.events"), + ], + ) + + +def test_ddb_and_bq_builds_validate_against_their_own_catalogs(): + errors, warnings = validate_iceberg_sink(_ddb_and_bq()) + assert errors == [] + # the only warning is the BigQuery build's published-sink advisory + assert len(warnings) == 1, warnings + assert "build 'ingest_events'" in warnings[0] + assert "iceberg.catalog.type=bigquery" in warnings[0] + for bid in ("ingest_orders", "ingest_events"): + assert iceberg_sink_preflight(_ddb_and_bq(), bid) is None + + +def test_ddb_build_pushes_the_dynamodb_catalog_with_its_derived_warehouse( + kafka_connect_mock, tmp_path: Path +): + contract = _ddb_and_bq() + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest_orders-sink"]["config"] + assert sink["iceberg.tables"] == "sales.orders" + assert sink["iceberg.catalog.catalog-impl"] == "org.apache.iceberg.aws.dynamodb.DynamoDbCatalog" + assert sink["iceberg.catalog.warehouse"] == "s3://lake/sales/orders/" + assert sink["iceberg.catalog.client.region"] == "us-east-1" + assert not any(k.startswith("iceberg.catalog.gcp.") for k in sink) + + +def test_bq_build_pushes_the_bigquery_catalog_with_its_project_and_location( + kafka_connect_mock, tmp_path: Path +): + contract = _ddb_and_bq() + assert execute_kafka_connect_build(contract["builds"][1], contract, tmp_path) == 0 + sink = kafka_connect_mock.connectors["src-ingest_events-sink"]["config"] + assert sink["iceberg.tables"] == "analytics.events" + assert sink["iceberg.catalog.type"] == "bigquery" + assert sink["iceberg.catalog.warehouse"] == "gs://gbkt" + assert sink["iceberg.catalog.gcp.bigquery.project-id"] == "acme-proj" + assert sink["iceberg.catalog.gcp.bigquery.location"] == "europe-west1" + assert "iceberg.catalog.client.region" not in sink + assert "iceberg.catalog.catalog-impl" not in sink + + +def test_a_build_naming_both_kinds_is_refused_before_any_rest_call( + kafka_connect_mock, tmp_path: Path +): + contract = _contract([DDB, BQ], [_kc_build("ingest", ["orders", "events"], "public.orders")]) + errors, _ = validate_iceberg_sink(contract) + assert any("name 2 of the Iceberg sink exposes" in e for e in errors), errors + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) != 0 + assert kafka_connect_mock.connectors == {} + + +def test_bq_build_without_project_is_refused_even_when_the_first_expose_is_complete(): + no_project = copy.deepcopy(BQ) + del no_project["binding"]["location"]["project"] + contract = _contract( + [DDB, no_project], [_kc_build("ingest_events", ["events"], "public.events")] + ) + errors, _ = validate_iceberg_sink(contract) + assert any("bigquery catalog requires binding.location.project" in e for e in errors), errors + + +# ── embedded Debezium over two exposes of one JDBC / DynamoDB database ────── + + +def _two_tables(kind: str, **extra: Any) -> List[Dict[str, Any]]: + common: Dict[str, Any] = { + "database": "sales", + "bucket": "lake", + "region": "us-east-1", + "catalog": kind, + } + if kind == "jdbc": + common["uri"] = _JDBC_URI + common.update(extra) + return [ + _expose("orders", "aws", table="orders", **common), + _expose("refunds", "aws", table="refunds", **common), + ] + + +def _props_path(contract: Dict[str, Any], tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" + + +@pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) +def test_debezium_writes_two_bucket_derived_exposes_of_one_database(tmp_path: Path, kind, outputs): + # Each expose derives ``s3://lake/sales/
/``: a per-table prefix of + # one bucket, as Glue's is, not two catalogs. + contract = _contract(_two_tables(kind), [_dbz_build(outputs)]) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None + ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + DebeziumRunner().run(ctx) # no server binary: fails after writing the config + text = _props_path(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.table-namespace=sales" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/orders/" in text + + +@pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) +def test_debezium_refuses_bucket_derived_exposes_in_two_buckets(tmp_path: Path, kind): + exposes = _two_tables(kind) + exposes[1]["binding"]["location"]["bucket"] = "other-lake" + contract = _contract(exposes, [_dbz_build(None)]) + message = "resolve to different catalogs (their warehouse differ)" + errors, _ = validate_iceberg_sink(contract) + assert any(message in e for e in errors), errors + result = DebeziumRunner().run( + build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + ) + assert result.state == RunState.FAILED + assert message in (result.error or "") + assert not _props_path(contract, tmp_path).exists() + + +def test_debezium_compares_a_warehouse_the_binding_sets_whole(): + exposes = _two_tables("jdbc") + exposes[0]["binding"]["location"]["warehouse"] = "s3://lake/a/" + exposes[1]["binding"]["location"]["warehouse"] = "s3://lake/b/" + errors, _ = validate_iceberg_sink(_contract(exposes, [_dbz_build(None)])) + assert any("their warehouse differ" in e for e in errors), errors + exposes[1]["binding"]["location"]["warehouse"] = "s3://lake/a/" + assert validate_iceberg_sink(_contract(exposes, [_dbz_build(None)])) == ([], []) + + +@pytest.mark.parametrize( + "key, value, named", + [("project", "other-proj", "project"), ("region", "us-central1", "region")], +) +def test_debezium_names_the_bigquery_setting_that_differs(key, value, named): + # ``location.project`` / ``location.region`` reach the catalog as + # gcp.bigquery.project-id / gcp.bigquery.location; the refusal names the + # binding setting. + first = copy.deepcopy(BQ) + first["exposeId"] = "orders" + second = copy.deepcopy(BQ) + second["exposeId"] = "refunds" + second["binding"]["location"]["table"] = "refunds" + second["binding"]["location"][key] = value + errors, _ = validate_iceberg_sink(_contract([first, second], [_dbz_build(None)])) + assert any(f"resolve to different catalogs (their {named} differ)" in e for e in errors), errors + + +# ── a Confluent Tableflow expose with no catalog, beside a sink, with grants ─ + +TABLEFLOW = _expose( + "tableflow_orders", + "confluent", + environment_id="env-123", + kafka_cluster_id="lkc-123", + topic="orders", + database="sales_glue", + bucket="acme-tableflow", + confluent_role_arn="arn:aws:iam::123456789012:role/tableflow", + region="eu-west-1", +) +GLUE = _expose("orders", "aws", database="sales", table="orders", bucket="lake", region="us-east-1") + + +def test_a_tableflow_expose_is_not_a_sink_candidate(): + # one self-managed Iceberg expose: a build with no outputs writes it + contract = _contract([TABLEFLOW, GLUE], [_kc_build("ingest", None, "public.orders")]) + assert validate_iceberg_sink(contract) == ([], []) + + +def test_a_derived_sink_naming_only_the_tableflow_expose_is_refused( + kafka_connect_mock, tmp_path: Path +): + contract = _contract( + [TABLEFLOW, GLUE], [_kc_build("ingest", ["tableflow_orders"], "public.orders")] + ) + errors, _ = validate_iceberg_sink(contract) + assert any( + "outputs ['tableflow_orders'] name none of the Iceberg sink exposes" in e for e in errors + ), errors + assert execute_kafka_connect_build(contract["builds"][0], contract, tmp_path) != 0 + assert kafka_connect_mock.connectors == {} + + +def test_policy_compile_grants_both_glue_tables_and_prints_no_warning(tmp_path: Path, caplog): + contract = _contract( + [TABLEFLOW, GLUE], [_kc_build("ingest", ["orders"], "public.orders")], grants=True + ) + bindings, warnings = compile_policy(contract) + assert warnings == [] + assert [(b["provider"], b["resource_type"], b["resource_id"]) for b in bindings] == [ + ("aws", "s3.bucket", "acme-tableflow"), + ("aws", "glue.table", "sales_glue"), + ("aws", "s3.bucket", "lake"), + ("aws", "glue.table", "sales.orders"), + ] + + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + out = tmp_path / "bindings.json" + logger = logging.getLogger("test.fw2.seams") + caplog.set_level(logging.DEBUG, logger="test.fw2.seams") + assert policy_compile.run(argparse.Namespace(contract=str(path), out=str(out)), logger) == 0 + assert json.loads(out.read_text())["bindings"] == bindings + shown = [r for r in caplog.records if r.levelno >= logging.WARNING] + assert shown == [], [r.getMessage() for r in shown] From 154175abcc91c64b09504949e8db696985ea380d Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:56:53 +0200 Subject: [PATCH 13/19] docs(changelog): the catalog-table, sink-expose and policy-guard fixes --- CHANGELOG.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54785486..9c1efd31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 table lived in two catalogs; `fluid validate` and the runners now refuse it unless the sink writes BigLake. Set `catalog: bigquery`, or the REST kind your catalog is. A hand-written sink config must then select that same catalog. +- An Iceberg expose with `catalog: dynamodb` or `catalog: jdbc` that a streaming sink + writes to now needs a warehouse: both catalogs refuse to start without one. Set + `location.warehouse`, or an explicit `location.bucket` on `platform: aws` or `gcp`, from + which forge-cli derives `s3://` or `gs://` `/` (`path` defaults to + `/
/`), or set the sink's `warehouse` property in an override. With none + of these, `fluid validate` and the runner preflight refuse the build. +- An Iceberg expose with `catalog: bigquery` that a streaming sink writes to now needs + `location.project`, which becomes the catalog's `gcp.bigquery.project-id`, or that + property in an override. A blank value counts as missing. +- The derived BigQuery sink config no longer sets `client.region`: `location.region` goes + to `gcp.bigquery.location`. Its warehouse is now the `gs://` storage URI dbt-bigquery and + the GCP IaC use (a `gs://` `location.warehouse`, else `gs://` plus + `location.path` when set, with the bucket's `{{ env.* }}` templates rendered), not + `location.warehouse` as written; a warehouse with another scheme, or none and no bucket, + gives none, and `fluid validate` warns. +- A `platform: confluent` Iceberg expose with no `location.catalog` now reads as Glue, the + catalog Tableflow publishes it to. `fluid policy compile` now emits its S3 bucket and + Glue table grants, and dbt-snowflake `catalogs.yml` writes + `catalog_linked_database_type: glue` for it. +- A Kafka Connect build whose Iceberg sink config forge-cli derives now writes exactly one + Iceberg expose, picked by its `outputs`. `fluid validate` and the run preflight refuse it + when the outputs name no Iceberg sink expose or name two or more, and when it declares + no outputs and the contract has two or more. Outputs that named no Iceberg sink expose + used to draw only the warning "the join is implicit". Split the build into one build per + expose, or hand-write `properties.kafka-connect.sink_connector_config`. +- An embedded Debezium Server build whose Iceberg sink config forge-cli derives now writes + the Iceberg sink exposes its `outputs` name, or all of them with no outputs. It is + refused when those exposes sit in more than one database, resolve to different catalogs, + or when the outputs name none of them. +- `fluid policy compile` now exits 1 with `policy_compiler_crashed` and writes no bindings + file when the policy compiler crashes. It used to write an empty bindings list and exit + 0, so the next stage read the crash as "no grants to enforce". ### Changed @@ -57,6 +89,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 it accepted. They ran beside `publish-pypi` and ignored its result, so a refused upload (for example "file already exists" on a re-dispatched tag) still cut a release and moved the GHCR `latest` tag. The release takes the quality gate's validated tag name. +- A derived Kafka Connect or embedded Debezium Server Iceberg sink resolves the exposes it + writes from the build's `outputs`, through one resolver that `fluid validate`, the run + preflight, both runners and the Kafka Connect late-arrival target read. The Debezium + Server sink writes every captured table under one `table-namespace` through one catalog, + so the exposes must share one `binding.location.database` and one catalog. A Glue + warehouse, and a DynamoDB or JDBC warehouse derived from `location.bucket`, is a + per-table prefix: Glue's is not compared, the derived one only by its bucket, and the + sink takes the first expose's. A hand-written config, and an override that sets + `iceberg.tables` (Kafka Connect) or `table-namespace` (Debezium Server), is still checked + against the first Iceberg sink expose. +- The Kafka Connect deriver now forwards the catalog's own properties (for BigQuery, + `gcp.bigquery.project-id` and `gcp.bigquery.location`) under the `iceberg.catalog.` + prefix, as the Debezium Server deriver already did. ### Fixed @@ -206,6 +251,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a name its SQL cannot quote all passed stage 2 and failed first at `fluid generate iac` or `fluid apply`. Both now run the same derivation (`lf_governance`) and refuse with the same message. +- **A second CDC build no longer streams into the first build's table.** A Kafka Connect + or embedded Debezium build whose sink config forge-cli derives always wrote the first + Iceberg expose, so a build for `refunds` pushed `iceberg.tables=sales.orders` while + `fluid validate`, the preflight and the run all passed. It now writes the expose its + `outputs` name (see Upgrade notes). +- **DynamoDB, JDBC and BigQuery sink configs carry what the catalog needs to start.** + `DynamoDbCatalog` and `JdbcCatalog` refuse to initialise without a warehouse, and + `BigQueryMetastoreCatalog` without `gcp.bigquery.project-id`; forge-cli derived neither + and `fluid validate` asked for neither, so the connector failed at start (see Upgrade + notes). A warehouse or project an override supplies counts, and a whitespace-only + `location.warehouse` or `location.project` counts as unset. +- **A bucket written as a `{{ env.* }}` template is read as written.** When its variable is + unset or empty where `fluid validate` runs, validate warns and names the variable instead + of asking for a bucket the contract already sets. The runner preflight, where the sink + config is derived, refuses the build instead of pushing a warehouse in a bucket such as + `acme--lake` that the contract does not name; this includes a run from a plan, whose + embedded contract is read as written. A Kafka Connect BigQuery sink reads the warehouse + only to auto-create tables, so it is refused only with auto-create on; otherwise it + warns and pushes no warehouse. An override warehouse is kept, and a hand-written sink + config that does not derive is not checked against the bucket's variables. +- **The BigQuery no-warehouse warning states what fails.** With auto-create on, the Kafka + Connect sink calls `createNamespace` for each table it creates, and + `BigQueryMetastoreCatalog` refuses that without a warehouse, so auto-create fails even in + a dataset that exists. For Debezium Server it says the server does not boot without + `debezium.sink.iceberg.warehouse`, which has no default. +- **`fluid policy compile` grants a Confluent Tableflow table with no catalog.** A + `platform: confluent` Iceberg expose with no catalog read as `rest`, so `fluid policy + compile` dropped its grants with a false "cataloged in 'rest'" warning (see Upgrade + notes). +- **`fluid policy compile` shows its warnings.** Each compiler warning, for example a grant + on an Iceberg table in a catalog it cannot grant on, which compiles to no table grant, is + now printed at WARNING level, and the exit code stays 0; the warnings are still written to + `bindings.json`. `fluid policy apply` echoes the warnings stored in `bindings.json`. A + contract with no grants prints no warning. +- **A policy compiler crash fails the command.** `fluid policy compile` and the policy step + of `fluid generate artifacts` now fail with exit 1 and `policy_compiler_crashed` (see + Upgrade notes). The error names the schema type errors at the values the compiler reads + (`accessPolicy`, its grants, each grant's permissions, each expose's binding and + location), or the compiler's own exception when the schema finds none there. +- **The Snowflake EXTERNAL VOLUME catalog-move guard names both causes.** It blocks the + same applies, but its message no longer claims forge-cli stopped + creating the volume or that the moved exposes held it. It names both possible causes (an + upgrade from a release that gave a non-Snowflake-managed catalog a volume, or a + Snowflake-managed expose removed or re-cataloged in this change) and keeps the `tofu + state rm` remediation. ### Documentation From 930b9f6cbc7742c912ba3d42cff02d24ec8494ac Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:36:23 +0200 Subject: [PATCH 14/19] fix(iceberg): grant the Glue table Tableflow publishes; refuse a Debezium sink over differing derived warehouses policy compile read only location.table for a Confluent Tableflow expose, so it granted a Glue table nothing publishes, or with no table a database-wide glue.table with table null. It now grants ., the table the Tableflow IaC names (_topic_name: topic > table > exposeId), in location.database or, with none, the Kafka cluster id database Tableflow defaults to; with neither it warns and grants no Glue table. An embedded Debezium sink over several DynamoDB / JDBC exposes used the first expose's bucket-derived table prefix as the catalog warehouse, so the catalog created every other table inside that prefix. Derived warehouses are compared whole again, and differing ones are refused by validate, the preflight and the runner, naming each expose's warehouse. --- fluid_build/build_runners/debezium/runner.py | 6 +- fluid_build/policy/compiler.py | 47 +++++ fluid_build/providers/_iceberg_catalog.py | 58 +++++-- .../test_fw2_debezium_derived_warehouse.py | 164 ++++++++++++++++++ .../test_fw2_integration_seams.py | 32 +++- .../test_fw2_confluent_default_catalog.py | 2 +- tests/policy/test_fw2_tableflow_glue_table.py | 138 +++++++++++++++ 7 files changed, 419 insertions(+), 28 deletions(-) create mode 100644 tests/build_runners/test_fw2_debezium_derived_warehouse.py create mode 100644 tests/policy/test_fw2_tableflow_glue_table.py diff --git a/fluid_build/build_runners/debezium/runner.py b/fluid_build/build_runners/debezium/runner.py index aafbce7e..bdfbb427 100644 --- a/fluid_build/build_runners/debezium/runner.py +++ b/fluid_build/build_runners/debezium/runner.py @@ -553,9 +553,9 @@ def _execute_debezium_server( # The exposes the build's outputs name, as `fluid validate` resolves # them. They share one database and one catalog, so the first one's - # binding gives the namespace and the catalog block (for Glue, and for - # a DynamoDB / JDBC warehouse derived from the bucket, whose warehouse - # is a per-table prefix, the first one's prefix). + # binding gives the namespace and the catalog block (for Glue, whose + # warehouse is a per-table prefix, the first one's prefix; a DynamoDB / + # JDBC warehouse is one value for all of them, or the resolver refuses). targets, _error = iceberg_sink_target(ctx.contract, build, sink_plan) if targets: resolved = resolve_iceberg_catalog( diff --git a/fluid_build/policy/compiler.py b/fluid_build/policy/compiler.py index 6622ec40..89ddc51e 100644 --- a/fluid_build/policy/compiler.py +++ b/fluid_build/policy/compiler.py @@ -99,6 +99,10 @@ def compile_policy(contract: dict) -> Tuple[List[Dict[str, Any]], List[str]]: _compile_catalog_managed_iceberg( bindings, warnings, exp, binding, principal, permissions ) + elif _is_tableflow(binding): + _compile_tableflow_bindings( + bindings, warnings, exp, fmt, loc, principal, permissions + ) elif platform == "aws" or fmt in ("s3_file", "iceberg", "parquet"): _compile_aws_bindings(bindings, fmt, loc, principal, permissions) elif platform == "snowflake" or fmt == "snowflake_table": @@ -222,6 +226,49 @@ def _compile_catalog_managed_iceberg(bindings, warnings, exp, binding, principal ) +def _is_tableflow(binding) -> bool: + """Is this a Glue-cataloged Iceberg expose the Confluent Tableflow IaC publishes? + + Matched the way that emitter matches its exposes (``is_cloud``), so a + platform alias it publishes is compiled here too. + """ + from ..iac.provider_match import is_cloud + from ..providers._iceberg_catalog import is_iceberg_format + + return is_cloud(binding, "confluent") and is_iceberg_format(binding.get("format")) + + +def _compile_tableflow_bindings(bindings, warnings, exp, fmt, loc, principal, permissions): + """Compile a Confluent Tableflow expose: its bucket and the Glue table it publishes. + + Tableflow publishes the table for the Kafka topic, not ``location.table``: + the Tableflow IaC names it with ``_topic_name`` (topic > table > exposeId) + and publishes it into ``location.database`` (``custom_database``). With no + database, Tableflow's Glue database is the Kafka cluster id ("By default, + the Glue database name is the cluster ID, and each topic becomes a table + under it": docs.confluent.io/cloud/current/topics/tableflow/how-to-guides/ + catalog-integration/integrate-with-aws-glue-catalog). When neither names + the database, the Glue grant is reported as a warning, not widened to a + database. + """ + from ..iac.providers.confluent import _topic_name + + topic = _topic_name(loc, exp) + database = loc.get("database") or loc.get("kafka_cluster_id") + if database: + published = {**loc, "database": database, "dataset": None, "table": topic} + _compile_aws_bindings(bindings, fmt, published, principal, permissions) + return + _compile_aws_bindings(bindings, fmt, loc, principal, permissions, glue=False) + expose_id = exp.get("exposeId") or exp.get("id") or "?" + warnings.append( + f"Iceberg expose '{expose_id}' (platform confluent) names neither " + f"binding.location.database nor kafka_cluster_id, so the Glue database Tableflow " + f"publishes topic '{topic}' into is unknown and no Glue table grant was compiled " + f"for {principal} {list(permissions)}." + ) + + def _compile_aws_bindings(bindings, fmt, loc, principal, permissions, *, glue=True): """Generate AWS IAM policy statements. diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 31032c5f..476881ff 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -433,6 +433,18 @@ def resolve_iceberg_sink_exposes( f"binding.location.database, split the build into one build per database, or {remedy}" ) differs = _catalog_differences(selected) + derived = _derived_warehouses(selected) if differs == ["warehouse"] else {} + if derived: + kind = resolve_iceberg_catalog(selected[0].get("binding") or {}).kind + return (), ( + f"{head}: a derived Debezium Server sink writes every table through one {kind} " + f"catalog warehouse, and the catalog creates a missing table under that " + f"warehouse ({_BUCKET_WAREHOUSE_LAYOUT[kind]}). But {writes} derive different " + f"warehouses from binding.location.bucket and path ({derived}): each is that " + f"expose's own table prefix, so one expose's tables would be created inside " + f"another expose's prefix. Set one binding.location.warehouse on them, split the " + f"build into one build per expose, or {remedy}" + ) if differs: return (), ( f"{head}: a derived Debezium Server sink writes through one catalog, but {writes} " @@ -471,11 +483,12 @@ def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: too (``get_iceberg_warehouse`` derives ``s3://{bucket}/{database}/{table}/``), while a REST catalog's warehouse is the catalog's name. A DynamoDB or JDBC warehouse derived from ``location.bucket`` (:func:`_bucket_warehouse`) is - a per-table prefix the same way, so only its bucket is compared. + compared whole: the catalog creates every missing table under the one + warehouse the sink is given, so it is not a per-table prefix. """ from dataclasses import fields as dataclass_fields - resolved = [_bucket_rooted(e.get("binding") or {}) for e in exposes] + resolved = [resolve_iceberg_catalog(e.get("binding") or {}) for e in exposes] if len(resolved) < 2: return [] per_table = {"fq_table", "id_columns", "partition_by"} @@ -504,21 +517,34 @@ def _catalog_differences(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: return differs -def _bucket_rooted(binding: Mapping[str, Any]) -> ResolvedIcebergCatalog: - """:func:`resolve_iceberg_catalog` of ``binding``, a bucket-derived warehouse cut to its bucket. +#: Where a catalog that is given a warehouse creates a table it does not have +#: (apache-iceberg-1.10.0: aws/.../dynamodb/DynamoDbCatalog.java:163-186, for +#: a namespace with no ``location`` property; core/.../jdbc/JdbcCatalog.java: +#: 282-284 and 521-526). Both strip the warehouse's trailing slash. +_BUCKET_WAREHOUSE_LAYOUT: Mapping[str, str] = MappingProxyType( + { + "dynamodb": "/.db/
", + "jdbc": "//
", + } +) + - ``s3://lake/sales/orders/`` becomes ``s3://lake``: the path - :func:`_bucket_warehouse` appends defaults to the table's own prefix. A - ``location.warehouse`` the binding sets is kept whole. - """ - from dataclasses import replace - - resolved = resolve_iceberg_catalog(binding) - explicit = str((binding.get("location") or {}).get("warehouse") or "").strip() - if resolved.kind not in BUCKET_WAREHOUSE_KINDS or explicit or "://" not in resolved.warehouse: - return resolved - scheme, rest = resolved.warehouse.split("://", 1) - return replace(resolved, warehouse=f"{scheme}://{rest.split('/', 1)[0]}") +def _derived_warehouses(exposes: Tuple[Mapping[str, Any], ...]) -> Dict[str, str]: + """``{exposeId: warehouse}`` when ``exposes`` are one DynamoDB or JDBC kind + whose warehouses :func:`_bucket_warehouse` derives (no + ``location.warehouse``) and differ, else ``{}``.""" + resolved = [resolve_iceberg_catalog(e.get("binding") or {}) for e in exposes] + kinds = {r.kind for r in resolved} + if len(kinds) != 1 or not kinds <= BUCKET_WAREHOUSE_KINDS: + return {} + if any( + str(((e.get("binding") or {}).get("location") or {}).get("warehouse") or "").strip() + for e in exposes + ): + return {} + if len({r.warehouse for r in resolved}) < 2: + return {} + return {str(_expose_id(e)): r.warehouse for e, r in zip(exposes, resolved, strict=True)} #: Object-store URI schemes, and the Iceberg ``FileIO`` each one needs. THE one diff --git a/tests/build_runners/test_fw2_debezium_derived_warehouse.py b/tests/build_runners/test_fw2_debezium_derived_warehouse.py new file mode 100644 index 00000000..ac711057 --- /dev/null +++ b/tests/build_runners/test_fw2_debezium_derived_warehouse.py @@ -0,0 +1,164 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An embedded Debezium sink over DynamoDB / JDBC exposes with derived warehouses. + +A warehouse derived from ``location.bucket`` and ``path`` is that expose's +own table prefix, and the catalog creates every missing table under the one +warehouse the sink is given (apache-iceberg-1.10.0 DynamoDbCatalog.java: +163-186, JdbcCatalog.java:282-284). Writing two such exposes through the +first one's prefix put the second one's data inside the first one's prefix, +so differing derived warehouses are refused. +""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.api.runner import RunState +from fluid_build.build_runners._acquisition_common import build_acquisition_run_context +from fluid_build.build_runners.debezium.runner import DebeziumRunner +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) + +pytestmark = [pytest.mark.unit] + +_JDBC_URI = "jdbc:postgresql://catalog-db:5432/iceberg" +_REFUSED = "derive different warehouses from binding.location.bucket and path" + + +def _expose(eid: str, kind: str, **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = { + "database": "sales", + "table": eid, + "bucket": "lake", + "region": "us-east-1", + "catalog": kind, + } + if kind == "jdbc": + loc["uri"] = _JDBC_URI + loc.update(location) + return { + "exposeId": eid, + "kind": "table", + "binding": {"platform": "aws", "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + + +def _contract(exposes: List[Dict[str, Any]], outputs: Optional[List[str]]) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": "ingest", + "pattern": "acquisition", + "engine": "debezium", + "capabilities": ["cdc", "streaming"], + "properties": { + "source": { + "kind": "postgres", + "connection": { + "host": "db", + "port": 5432, + "database": "app", + "user": "u", + "password": "p", + }, + "mode": "cdc", + "streams": [f"public.{e['exposeId']}" for e in exposes], + "watermark": {"strategy": "high_water_mark", "allowedLateness": "PT5M"}, + }, + "sink": {"format": "iceberg"}, + "debezium": {"deployment": {"mode": "embedded"}, "server": {"sink": {}}}, + }, + } + if outputs is not None: + build["outputs"] = outputs + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.cdc", + "name": "Sales CDC", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "x@y.z"}}, + "exposes": copy.deepcopy(exposes), + "builds": [build], + } + + +def _props(contract: Dict[str, Any], tmp_path: Path) -> Path: + return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" + + +def _run(contract: Dict[str, Any], tmp_path: Path): + ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + return DebeziumRunner().run(ctx) + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_a_restricted_expose_is_not_written_under_a_public_prefix(tmp_path: Path, kind): + contract = _contract( + [ + _expose("orders", kind, path="public/orders/"), + _expose("payroll", kind, path="restricted/payroll/"), + ], + ["orders", "payroll"], + ) + errors, _ = validate_iceberg_sink(contract) + refusal = next((e for e in errors if _REFUSED in e), "") + assert "'orders': 's3://lake/public/orders/'" in refusal, errors + assert "'payroll': 's3://lake/restricted/payroll/'" in refusal, errors + assert "Set one binding.location.warehouse on them" in refusal + assert _REFUSED in (iceberg_sink_preflight(contract, "ingest") or "") + result = _run(contract, tmp_path) + assert result.state == RunState.FAILED + assert _REFUSED in (result.error or "") + assert not _props(contract, tmp_path).exists() + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +@pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) +def test_default_paths_of_one_database_are_refused(tmp_path: Path, kind, outputs): + # ``path`` defaults to ``/
/``: sales/orders/ and sales/refunds/. + contract = _contract([_expose("orders", kind), _expose("refunds", kind)], outputs) + errors, _ = validate_iceberg_sink(contract) + assert any(_REFUSED in e for e in errors), errors + assert _run(contract, tmp_path).state == RunState.FAILED + assert not _props(contract, tmp_path).exists() + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_one_shared_path_derives_one_warehouse(tmp_path: Path, kind): + contract = _contract( + [_expose("orders", kind, path="sales/"), _expose("refunds", kind, path="sales/")], + None, + ) + assert validate_iceberg_sink(contract) == ([], []) + assert iceberg_sink_preflight(contract, "ingest") is None + _run(contract, tmp_path) # no server binary: fails after writing the config + text = _props(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/" in text + assert "debezium.sink.iceberg.table-namespace=sales" in text + + +@pytest.mark.parametrize("kind", ["dynamodb", "jdbc"]) +def test_a_single_expose_keeps_its_derived_warehouse(tmp_path: Path, kind): + contract = _contract([_expose("orders", kind)], ["orders"]) + assert validate_iceberg_sink(contract) == ([], []) + _run(contract, tmp_path) + text = _props(contract, tmp_path).read_text() + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/orders/" in text diff --git a/tests/build_runners/test_fw2_integration_seams.py b/tests/build_runners/test_fw2_integration_seams.py index fe9511a7..a814fa1d 100644 --- a/tests/build_runners/test_fw2_integration_seams.py +++ b/tests/build_runners/test_fw2_integration_seams.py @@ -245,19 +245,36 @@ def _props_path(contract: Dict[str, Any], tmp_path: Path) -> Path: return tmp_path / ".fluid" / "debezium" / contract["id"] / "ingest" / "application.properties" +_DERIVED = "derive different warehouses from binding.location.bucket and path" + + @pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) @pytest.mark.parametrize("outputs", [["orders", "refunds"], None], ids=["both", "no-outputs"]) -def test_debezium_writes_two_bucket_derived_exposes_of_one_database(tmp_path: Path, kind, outputs): - # Each expose derives ``s3://lake/sales/
/``: a per-table prefix of - # one bucket, as Glue's is, not two catalogs. +def test_debezium_refuses_two_bucket_derived_exposes_of_one_database(tmp_path: Path, kind, outputs): + # Each expose derives ``s3://lake/sales/
/``, its own table prefix. + # The catalog creates every missing table under the one warehouse the + # sink is given, so writing both through either prefix is refused. contract = _contract(_two_tables(kind), [_dbz_build(outputs)]) + errors, _ = validate_iceberg_sink(contract) + assert any(_DERIVED in e for e in errors), errors + assert _DERIVED in (iceberg_sink_preflight(contract, "ingest") or "") + result = DebeziumRunner().run( + build_acquisition_run_context(contract["builds"][0], contract, tmp_path) + ) + assert result.state == RunState.FAILED + assert not _props_path(contract, tmp_path).exists() + + +@pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) +def test_debezium_writes_two_exposes_of_one_database_under_one_warehouse(tmp_path: Path, kind): + contract = _contract(_two_tables(kind, warehouse="s3://lake/sales/"), [_dbz_build(None)]) assert validate_iceberg_sink(contract) == ([], []) assert iceberg_sink_preflight(contract, "ingest") is None ctx = build_acquisition_run_context(contract["builds"][0], contract, tmp_path) DebeziumRunner().run(ctx) # no server binary: fails after writing the config text = _props_path(contract, tmp_path).read_text() assert "debezium.sink.iceberg.table-namespace=sales" in text - assert "debezium.sink.iceberg.warehouse=s3://lake/sales/orders/" in text + assert "debezium.sink.iceberg.warehouse=s3://lake/sales/" in text @pytest.mark.parametrize("kind", ["jdbc", "dynamodb"]) @@ -265,14 +282,13 @@ def test_debezium_refuses_bucket_derived_exposes_in_two_buckets(tmp_path: Path, exposes = _two_tables(kind) exposes[1]["binding"]["location"]["bucket"] = "other-lake" contract = _contract(exposes, [_dbz_build(None)]) - message = "resolve to different catalogs (their warehouse differ)" errors, _ = validate_iceberg_sink(contract) - assert any(message in e for e in errors), errors + assert any(_DERIVED in e for e in errors), errors result = DebeziumRunner().run( build_acquisition_run_context(contract["builds"][0], contract, tmp_path) ) assert result.state == RunState.FAILED - assert message in (result.error or "") + assert _DERIVED in (result.error or "") assert not _props_path(contract, tmp_path).exists() @@ -348,7 +364,7 @@ def test_policy_compile_grants_both_glue_tables_and_prints_no_warning(tmp_path: assert warnings == [] assert [(b["provider"], b["resource_type"], b["resource_id"]) for b in bindings] == [ ("aws", "s3.bucket", "acme-tableflow"), - ("aws", "glue.table", "sales_glue"), + ("aws", "glue.table", "sales_glue.orders"), ("aws", "s3.bucket", "lake"), ("aws", "glue.table", "sales.orders"), ] diff --git a/tests/policy/test_fw2_confluent_default_catalog.py b/tests/policy/test_fw2_confluent_default_catalog.py index 3a4ec1bd..df28789b 100644 --- a/tests/policy/test_fw2_confluent_default_catalog.py +++ b/tests/policy/test_fw2_confluent_default_catalog.py @@ -73,7 +73,7 @@ def test_policy_compile_grants_the_glue_table_when_catalog_is_absent(): assert not any("cataloged in" in w for w in warnings), warnings assert [(b["provider"], b["resource_type"], b["resource_id"]) for b in bindings] == [ ("aws", "s3.bucket", "acme-tableflow"), - ("aws", "glue.table", "sales_glue"), + ("aws", "glue.table", "sales_glue.orders"), ] diff --git a/tests/policy/test_fw2_tableflow_glue_table.py b/tests/policy/test_fw2_tableflow_glue_table.py new file mode 100644 index 00000000..a3f24f44 --- /dev/null +++ b/tests/policy/test_fw2_tableflow_glue_table.py @@ -0,0 +1,138 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Confluent Tableflow expose is granted the Glue table Tableflow publishes. + +The Tableflow IaC names that table for the Kafka topic (``_topic_name``: +topic > table > exposeId) in ``location.database``, or with no database in +the database Tableflow names after the Kafka cluster id. The compiler used +to read only ``location.table``: a grant on a table nothing publishes, or, +with no table, a database-wide ``glue.table`` grant with ``table: null``. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Sequence + +import pytest + +from fluid_build.iac.providers.confluent import ConfluentIacPlugin +from fluid_build.policy.compiler import SAFE_GLUE_PERMS, compile_policy + +pytestmark = [pytest.mark.unit] + +_LOCATION: Dict[str, Any] = { + "environment_id": "env-123", + "kafka_cluster_id": "lkc-123", + "database": "sales_glue", + "bucket": "acme-tableflow", + "confluent_role_arn": "arn:aws:iam::123456789012:role/tableflow", + "region": "eu-west-1", +} + + +def _contract(permissions: Sequence[str] = ("read",), **location: Any) -> Dict[str, Any]: + loc = {**_LOCATION, **location} + loc = {k: v for k, v in loc.items() if v is not None} + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.tableflow", + "name": "Tableflow orders", + "metadata": {"layer": "Bronze"}, + "accessPolicy": { + "grants": [ + {"principal": "group:analysts@example.com", "permissions": list(permissions)} + ] + }, + "exposes": [ + { + "exposeId": "tableflow_orders", + "kind": "table", + "binding": {"platform": "confluent", "format": "iceberg", "location": loc}, + "contract": {"schema": [{"name": "order_id", "type": "integer"}]}, + } + ], + } + + +def _glue(bindings: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return [b for b in bindings if b["resource_type"] == "glue.table"] + + +def _published_topic(contract: Dict[str, Any]) -> str: + resources = ConfluentIacPlugin().emit(contract) + return next(iter(resources["confluent_tableflow_topic"].values()))["display_name"] + + +@pytest.mark.parametrize( + "location, table", + [ + ({"topic": "orders.v1", "table": "orders"}, "orders.v1"), + ({"topic": "orders"}, "orders"), + ({"table": "orders"}, "orders"), + ({}, "tableflow_orders"), + ], + ids=["topic-and-table", "topic-only", "table-only", "neither"], +) +def test_the_glue_grant_names_the_table_tableflow_publishes(location, table): + contract = _contract(**location) + bindings, warnings = compile_policy(contract) + assert warnings == [] + assert _published_topic(contract) == table + [glue] = _glue(bindings) + assert (glue["resource_id"], glue["database"], glue["table"]) == ( + f"sales_glue.{table}", + "sales_glue", + table, + ) + assert glue["actions"] == SAFE_GLUE_PERMS["readData"] + assert [b["resource_id"] for b in bindings if b["resource_type"] == "s3.bucket"] == [ + "acme-tableflow" + ] + + +def test_no_database_grants_the_cluster_id_database(): + bindings, warnings = compile_policy(_contract(topic="orders", database=None)) + assert warnings == [] + [glue] = _glue(bindings) + assert (glue["resource_id"], glue["database"], glue["table"]) == ( + "lkc-123.orders", + "lkc-123", + "orders", + ) + + +def test_no_database_and_no_cluster_id_warns_instead_of_granting_a_database(): + bindings, warnings = compile_policy( + _contract(topic="orders", database=None, kafka_cluster_id=None) + ) + assert _glue(bindings) == [] + assert [b["resource_type"] for b in bindings] == ["s3.bucket"] + assert len(warnings) == 1 + assert "'tableflow_orders'" in warnings[0] + assert "no Glue table grant was compiled" in warnings[0] + + +def test_a_write_grant_manages_the_published_table(): + bindings, _ = compile_policy(_contract(permissions=["write"], topic="orders")) + [glue] = _glue(bindings) + assert glue["resource_id"] == "sales_glue.orders" + assert glue["actions"] == SAFE_GLUE_PERMS["manage"] + + +def test_no_tableflow_glue_grant_has_a_null_table(): + for location in ({}, {"topic": "orders"}, {"database": None}): + bindings, _ = compile_policy(_contract(**location)) + assert all(b["table"] for b in _glue(bindings)), bindings From cecf103f5529881a61e5b66f1416734bf7dfcd16 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:48:55 +0200 Subject: [PATCH 15/19] docs(changelog): Tableflow grants name the topic table; differing derived DynamoDB/JDBC warehouses are refused --- CHANGELOG.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c1efd31..9313f174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,9 +45,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `location.warehouse` as written; a warehouse with another scheme, or none and no bucket, gives none, and `fluid validate` warns. - A `platform: confluent` Iceberg expose with no `location.catalog` now reads as Glue, the - catalog Tableflow publishes it to. `fluid policy compile` now emits its S3 bucket and - Glue table grants, and dbt-snowflake `catalogs.yml` writes - `catalog_linked_database_type: glue` for it. + catalog Tableflow publishes it to, and dbt-snowflake `catalogs.yml` writes + `catalog_linked_database_type: glue` for it. `fluid policy compile` grants a Confluent + Tableflow expose its S3 bucket and the Glue table Tableflow publishes: the topic's table + (`topic`, else `location.table`, else the expose id, as the Tableflow IaC names it) in + `location.database`, or, with none, in the Kafka cluster id database Tableflow defaults + to. With neither database nor `kafka_cluster_id` it warns and grants no Glue table. - A Kafka Connect build whose Iceberg sink config forge-cli derives now writes exactly one Iceberg expose, picked by its `outputs`. `fluid validate` and the run preflight refuse it when the outputs name no Iceberg sink expose or name two or more, and when it declares @@ -94,9 +97,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 preflight, both runners and the Kafka Connect late-arrival target read. The Debezium Server sink writes every captured table under one `table-namespace` through one catalog, so the exposes must share one `binding.location.database` and one catalog. A Glue - warehouse, and a DynamoDB or JDBC warehouse derived from `location.bucket`, is a - per-table prefix: Glue's is not compared, the derived one only by its bucket, and the - sink takes the first expose's. A hand-written config, and an override that sets + warehouse is a per-table prefix: it is not compared, and the sink takes the first + expose's. A DynamoDB or JDBC catalog creates every missing table under the one warehouse + the sink is given, so warehouses derived from `location.bucket` that differ (each is that + expose's own table prefix) are refused; set one `location.warehouse` on those exposes, or + split the build. A hand-written config, and an override that sets `iceberg.tables` (Kafka Connect) or `table-namespace` (Debezium Server), is still checked against the first Iceberg sink expose. - The Kafka Connect deriver now forwards the catalog's own properties (for BigQuery, From 2ed842b2d04281ebc603c8cc472df7be1cd2e23c Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:04:47 +0200 Subject: [PATCH 16/19] fix(policy): the compiler crash error names a wrong-typed value's path, never the value The schema type error policy_compiler_crashed printed included the value as written, so a connection URL with a password in a binding's location reached stderr and CI logs. It now names the path and the expected type only. --- CHANGELOG.md | 4 +++- fluid_build/cli/policy_compile.py | 7 ++++-- .../test_fw2_policy_compile_read_blocks.py | 23 +++++++++++++++---- .../test_fw2_policy_compile_schema_error.py | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9313f174..48def5d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -294,7 +294,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 of `fluid generate artifacts` now fail with exit 1 and `policy_compiler_crashed` (see Upgrade notes). The error names the schema type errors at the values the compiler reads (`accessPolicy`, its grants, each grant's permissions, each expose's binding and - location), or the compiler's own exception when the schema finds none there. + location) by path and expected type, never by value (a value of the wrong type can be a + connection URL with a password in it), or the compiler's own exception when the schema + finds none there. - **The Snowflake EXTERNAL VOLUME catalog-move guard names both causes.** It blocks the same applies, but its message no longer claims forge-cli stopped creating the volume or that the moved exposes held it. It names both possible causes (an diff --git a/fluid_build/cli/policy_compile.py b/fluid_build/cli/policy_compile.py index b5883422..df2823c7 100644 --- a/fluid_build/cli/policy_compile.py +++ b/fluid_build/cli/policy_compile.py @@ -60,7 +60,7 @@ def register(subparsers: argparse._SubParsersAction): def _compiler_schema_errors(contract: Any) -> List[str]: - """The schema type errors ``fluid validate`` reports where the compiler reads. + """The schema type errors where the compiler reads, as `` is not of type ``. ``policy compile`` does not schema-validate, so a grant or binding of the wrong type reaches the compiler and fails there as a Python error that @@ -79,7 +79,10 @@ def _compiler_schema_errors(contract: Any) -> List[str]: for error in result.errors: path, _, message = error.partition(": ") if _COMPILER_READ_PATHS.fullmatch(path) and " is not of type " in message: - found.append(error) + # The path and the expected type, never the value: a value of the + # wrong type can be a connection URL with a password in it. + expected = message.rsplit(" is not of type ", 1)[1] + found.append(f"{path} is not of type {expected}") return found diff --git a/tests/cli/test_fw2_policy_compile_read_blocks.py b/tests/cli/test_fw2_policy_compile_read_blocks.py index 09f168da..c86d55f5 100644 --- a/tests/cli/test_fw2_policy_compile_read_blocks.py +++ b/tests/cli/test_fw2_policy_compile_read_blocks.py @@ -65,7 +65,7 @@ ], } -_BINDING_ERROR = "exposes[0].binding: 'nope' is not of type 'object'" +_BINDING_ERROR = "exposes[0].binding is not of type 'object'" def _write(tmp_path: Path, doc: Dict[str, Any]) -> Path: @@ -122,19 +122,19 @@ def test_an_unrelated_grant_error_is_not_named(tmp_path): [ ( lambda d: d.update(accessPolicy=["x"]), - "accessPolicy: ['x'] is not of type 'object'", + "accessPolicy is not of type 'object'", ), ( lambda d: d["accessPolicy"].update(grants="g"), - "accessPolicy.grants: 'g' is not of type 'array'", + "accessPolicy.grants is not of type 'array'", ), ( lambda d: d["accessPolicy"]["grants"][0].update(permissions=None), - "accessPolicy.grants[0].permissions: None is not of type 'array'", + "accessPolicy.grants[0].permissions is not of type 'array'", ), ( lambda d: d["exposes"][0]["binding"].update(location=None), - "exposes[0].binding.location: None is not of type 'object'", + "exposes[0].binding.location is not of type 'object'", ), ], ids=["accessPolicy", "grants", "permissions", "location"], @@ -212,3 +212,16 @@ def test_grants_that_compile_to_nothing_still_warn(tmp_path, caplog): shown = [e.get("warning") for e in _warning_events(caplog)] assert "No IAM bindings generated from contract" in shown + + +def test_the_crash_error_names_the_path_and_never_the_value(tmp_path): + """A wrong-typed value can be a connection URL with a password in it.""" + secret = "Sup3rS3cretPw" + doc = copy.deepcopy(_CONTRACT) + doc["exposes"][0]["binding"]["location"] = f"snowflake://svc_user:{secret}@xy12345/DB/S/T" + + error = _compile_error(tmp_path, doc) + + assert "exposes[0].binding.location is not of type 'object'" in error + assert secret not in error + assert "snowflake://" not in error diff --git a/tests/cli/test_fw2_policy_compile_schema_error.py b/tests/cli/test_fw2_policy_compile_schema_error.py index 2c8ebee3..442152ca 100644 --- a/tests/cli/test_fw2_policy_compile_schema_error.py +++ b/tests/cli/test_fw2_policy_compile_schema_error.py @@ -81,7 +81,7 @@ def test_a_malformed_grant_is_reported_as_the_schema_error(tmp_path): assert err.event == "policy_compiler_crashed" assert err.context["error"] == ( "policy compile failed on contract values of the wrong type: " - "accessPolicy.grants[0]: 'group:analysts@example.com' is not of type 'object'" + "accessPolicy.grants[0] is not of type 'object'" ) assert "AttributeError" not in err.context["error"] assert not (tmp_path / "o" / "b.json").exists() From acec235e0679a80430f755bb528095c2f5995e42 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:27:38 +0200 Subject: [PATCH 17/19] fix(iceberg): a GCP sink writes into the expose's BigQuery dataset The sink named its table's namespace from location.database only. A GCP binding names its dataset in location.dataset, so a BigQuery or other GCP Iceberg sink pushed iceberg.tables=None.
and validate passed. The namespace is now location.database, else location.dataset (the BigQuery metastore catalog maps a namespace to a dataset, apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:149, :200, :307). A derived sink whose expose names no namespace, or a derived Kafka Connect sink with no table, is refused by validate and the run preflight. --- CHANGELOG.md | 14 ++ .../kafka_connect/iceberg_sink_validation.py | 21 +++ .../build_runners/kafka_connect/runner.py | 7 +- fluid_build/providers/_iceberg_catalog.py | 21 ++- .../test_fw2_gcp_dataset_namespace.py | 152 ++++++++++++++++++ 5 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 tests/build_runners/test_fw2_gcp_dataset_namespace.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 48def5d2..ef30cb78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `fluid policy compile` now exits 1 with `policy_compiler_crashed` and writes no bindings file when the policy compiler crashes. It used to write an empty bindings list and exit 0, so the next stage read the crash as "no grants to enforce". +- A Kafka Connect or embedded Debezium build whose sink config forge-cli derives is now + refused when its expose names no table namespace (`location.database`, or on GCP + `location.dataset`), and a Kafka Connect one when its expose names no `location.table`. + The derived config used to name the table `None.
`, or leave Debezium Server on its + default namespace. An override that sets `iceberg.tables` or `table-namespace` itself is + not affected. ### Changed @@ -261,6 +267,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Iceberg expose, so a build for `refunds` pushed `iceberg.tables=sales.orders` while `fluid validate`, the preflight and the run all passed. It now writes the expose its `outputs` name (see Upgrade notes). +- **A GCP streaming sink writes into the expose's BigQuery dataset.** The sink named the + table's namespace from `location.database` only, but a GCP binding names its dataset in + `location.dataset`, so a BigQuery or other GCP Iceberg sink pushed + `iceberg.tables=None.
` and `fluid validate` passed. The namespace is now + `location.database`, else `location.dataset` (the BigQuery metastore catalog maps a + namespace to a dataset), and a sink with neither is refused (see Upgrade notes). The + Kafka Connect late-events table and the Debezium shared-database check read it the + same way. - **DynamoDB, JDBC and BigQuery sink configs carry what the catalog needs to start.** `DynamoDbCatalog` and `JdbcCatalog` refuse to initialise without a warehouse, and `BigQueryMetastoreCatalog` without `gcp.bigquery.project-id`; forge-cli derived neither diff --git a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py index afeb722a..91455200 100644 --- a/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py +++ b/fluid_build/build_runners/kafka_connect/iceberg_sink_validation.py @@ -426,6 +426,27 @@ def _check_catalog( auto_creates: bool = False, ) -> None: """The catalog checks (4-6), every one read off the kind's table row.""" + # The table the derived config names: ``.
`` for Kafka + # Connect's iceberg.tables, the namespace alone for Debezium Server's + # table-namespace. Without one the derived config named the table + # ``None.
`` (or Debezium Server's default namespace). HARD. + if derives_table_identity(runtime): + from ...providers._iceberg_catalog import iceberg_namespace + + location = binding.get("location") or {} + if not iceberg_namespace(binding): + errors.append( + f"iceberg sink (build {bid!r}): the expose's binding names no table namespace, " + "so the derived sink config has no namespace to write under. Set " + "binding.location.database (on GCP, binding.location.dataset)" + ) + elif runtime.engine == "kafka-connect" and not ( + location.get("table") or binding.get("table") + ): + errors.append( + f"iceberg sink (build {bid!r}): the expose's binding names no table, so the " + "derived iceberg.tables has none. Set binding.location.table" + ) loc = binding.get("location") or {} kind = iceberg_catalog_kind(binding, sink) info = catalog_kind_info(kind) diff --git a/fluid_build/build_runners/kafka_connect/runner.py b/fluid_build/build_runners/kafka_connect/runner.py index db9495c0..7ed93e08 100644 --- a/fluid_build/build_runners/kafka_connect/runner.py +++ b/fluid_build/build_runners/kafka_connect/runner.py @@ -344,9 +344,12 @@ def _execute(ctx: RunContext, runner: KafkaConnectRunner) -> RunResult: late_arrival_target = connector_name_for_topic if str(ctx.sink.format or "").lower() == "iceberg": _ib = _find_iceberg_expose_binding(ctx.contract, build) or {} + from ...providers._iceberg_catalog import iceberg_namespace + _loc = _ib.get("location") or {} - if _loc.get("database") and _loc.get("table"): - late_arrival_target = f"{_loc['database']}.{_loc['table']}" + _namespace = iceberg_namespace(_ib) + if _namespace and _loc.get("table"): + late_arrival_target = f"{_namespace}.{_loc['table']}" late_arrival_policy = extract_late_arrival_policy( contract_or_source=ctx.source, diff --git a/fluid_build/providers/_iceberg_catalog.py b/fluid_build/providers/_iceberg_catalog.py index 476881ff..003923de 100644 --- a/fluid_build/providers/_iceberg_catalog.py +++ b/fluid_build/providers/_iceberg_catalog.py @@ -352,10 +352,22 @@ def _expose_id(expose: Mapping[str, Any]) -> Any: return expose.get("exposeId") or expose.get("id") +def iceberg_namespace(binding: Mapping[str, Any]) -> str: + """The Iceberg namespace a sink writes ``binding``'s table under, or ``""``. + + ``location.database``, else a top-level ``database``, else + ``location.dataset``: a GCP binding names its BigQuery dataset there, and + the BigQuery metastore catalog maps a namespace to a dataset + (apache-iceberg-1.10.0 BigQueryMetastoreCatalog.java:149, :200, :307). + """ + loc = binding.get("location") or {} + value = loc.get("database") or binding.get("database") or loc.get("dataset") + return str(value or "").strip() + + def _expose_database(expose: Mapping[str, Any]) -> Any: """The database leg of an expose's table, read as :func:`resolve_iceberg_catalog` reads it.""" - binding = expose.get("binding") or {} - return (binding.get("location") or {}).get("database") or binding.get("database") + return iceberg_namespace(expose.get("binding") or {}) or None def _described(exposes: Tuple[Mapping[str, Any], ...]) -> list[str]: @@ -430,7 +442,8 @@ def resolve_iceberg_sink_exposes( return (), ( f"{head}: a derived Debezium Server sink writes every captured table under one " f"table-namespace, but {writes} sit in the databases {databases}. Give them one " - f"binding.location.database, split the build into one build per database, or {remedy}" + f"binding.location.database (location.dataset on GCP), split the build into one " + f"build per database, or {remedy}" ) differs = _catalog_differences(selected) derived = _derived_warehouses(selected) if differs == ["warehouse"] else {} @@ -626,7 +639,7 @@ def resolve_iceberg_catalog( and ``location.region`` as its own catalog properties. """ loc = binding.get("location") or {} - database = loc.get("database") or binding.get("database") + database = iceberg_namespace(binding) or None table = loc.get("table") or binding.get("table") fq_table = f"{database}.{table}" kind = iceberg_catalog_kind(binding, sink) diff --git a/tests/build_runners/test_fw2_gcp_dataset_namespace.py b/tests/build_runners/test_fw2_gcp_dataset_namespace.py new file mode 100644 index 00000000..ba001a85 --- /dev/null +++ b/tests/build_runners/test_fw2_gcp_dataset_namespace.py @@ -0,0 +1,152 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A derived sink names its table's namespace from ``location.dataset`` on GCP. + +A GCP binding names its BigQuery dataset in ``location.dataset``. The sink +read only ``location.database``, so it wrote ``iceberg.tables=None.
`` +and ``fluid validate`` passed. A derived sink whose expose names no namespace +is now refused. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.build_runners.debezium.iceberg_sink import emit_debezium_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink import emit_iceberg_sink_config +from fluid_build.build_runners.kafka_connect.iceberg_sink_validation import ( + iceberg_sink_preflight, + validate_iceberg_sink, +) +from fluid_build.providers._iceberg_catalog import resolve_iceberg_catalog + +pytestmark = [pytest.mark.unit] + + +def _gcp(table: Optional[str] = "events", **location: Any) -> Dict[str, Any]: + loc: Dict[str, Any] = {"catalog": "bigquery", "project": "acme-proj", "bucket": "lake"} + if table is not None: + loc["table"] = table + loc.update(location) + return {"platform": "gcp", "format": "iceberg", "location": loc} + + +def _contract( + bindings: List[Dict[str, Any]], + engine: str, + *, + overrides: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + props: Dict[str, Any] = { + "source": {"kind": "postgres", "mode": "incremental_append", "streams": ["public.o"]}, + "sink": {"format": "iceberg"}, + } + if engine == "debezium": + props["debezium"] = {"deployment": {"mode": "embedded"}, "server": {"sink": {}}} + else: + props["kafka-connect"] = { + "deployment": {"mode": "bring-your-own", "server_url": "http://c:8083"} + } + if overrides: + props["kafka-connect"]["iceberg_catalog_overrides"] = dict(overrides) + exposes = [ + { + "exposeId": f"e{i}", + "kind": "table", + "binding": copy.deepcopy(b), + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + for i, b in enumerate(bindings) + ] + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.o", + "name": "o", + "metadata": {"layer": "Bronze", "owner": {"team": "dp"}}, + "exposes": exposes, + "builds": [ + { + "id": "b1", + "pattern": "acquisition", + "engine": engine, + "outputs": [e["exposeId"] for e in exposes], + "properties": props, + } + ], + } + + +def _namespace_errors(errors: List[str]) -> List[str]: + return [e for e in errors if "names no table namespace" in e] + + +def test_a_gcp_dataset_names_the_kafka_connect_table(): + resolved = resolve_iceberg_catalog(_gcp(dataset="analytics")) + assert resolved.fq_table == "analytics.events" + cfg = emit_iceberg_sink_config(resolved, product_id="bronze.o", topics=["public.o"]) + assert cfg["iceberg.tables"] == "analytics.events" + + +def test_a_gcp_dataset_names_the_debezium_namespace(): + resolved = resolve_iceberg_catalog(_gcp(dataset="analytics")) + assert emit_debezium_iceberg_sink_config(resolved)["table-namespace"] == "analytics" + + +def test_location_database_wins_over_dataset(): + resolved = resolve_iceberg_catalog(_gcp(database="raw", dataset="analytics")) + assert resolved.fq_table == "raw.events" + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_gcp_dataset_validates(engine): + errors, _warnings = validate_iceberg_sink(_contract([_gcp(dataset="analytics")], engine)) + assert _namespace_errors(errors) == [], errors + + +@pytest.mark.parametrize("engine", ["kafka-connect", "debezium"]) +def test_a_derived_sink_with_no_namespace_is_refused(engine): + contract = _contract([_gcp()], engine) + errors, _warnings = validate_iceberg_sink(contract) + assert len(_namespace_errors(errors)) == 1, errors + assert "binding.location.dataset" in _namespace_errors(errors)[0] + assert iceberg_sink_preflight(contract, "b1") is not None + + +def test_a_derived_kafka_connect_sink_with_no_table_is_refused(): + errors, _warnings = validate_iceberg_sink( + _contract([_gcp(table=None, dataset="analytics")], "kafka-connect") + ) + assert any("names no table, so the derived iceberg.tables" in e for e in errors), errors + + +def test_an_override_that_names_the_tables_is_not_refused(): + contract = _contract( + [_gcp()], "kafka-connect", overrides={"iceberg.tables": "analytics.events"} + ) + errors, _warnings = validate_iceberg_sink(contract) + assert _namespace_errors(errors) == [], errors + + +def test_debezium_exposes_sharing_one_dataset_are_one_namespace(): + contract = _contract( + [_gcp(dataset="analytics"), _gcp(table="refunds", dataset="analytics")], + "debezium", + ) + errors, _warnings = validate_iceberg_sink(contract) + assert not [e for e in errors if "databases" in e or "names no table" in e], errors From 9f17ff1881962254fe96494422ae659a73e04bdc Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:38:37 +0200 Subject: [PATCH 18/19] test(policy): mark the fake password in the crash-error test as an allowlisted fixture --- tests/cli/test_fw2_policy_compile_read_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/test_fw2_policy_compile_read_blocks.py b/tests/cli/test_fw2_policy_compile_read_blocks.py index c86d55f5..e2dd1308 100644 --- a/tests/cli/test_fw2_policy_compile_read_blocks.py +++ b/tests/cli/test_fw2_policy_compile_read_blocks.py @@ -216,7 +216,7 @@ def test_grants_that_compile_to_nothing_still_warn(tmp_path, caplog): def test_the_crash_error_names_the_path_and_never_the_value(tmp_path): """A wrong-typed value can be a connection URL with a password in it.""" - secret = "Sup3rS3cretPw" + secret = "Sup3rS3cretPw" # pragma: allowlist secret doc = copy.deepcopy(_CONTRACT) doc["exposes"][0]["binding"]["location"] = f"snowflake://svc_user:{secret}@xy12345/DB/S/T" From 7e73e2fc773cd3d93e5b5f3fe3c68866d7d4ce00 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:48:23 +0200 Subject: [PATCH 19/19] fix(deps): raise the mcp floor to 1.22; 1.20 and 1.21 no longer import with pydantic 2.14 mcp 1.20 and 1.21 import eval_type_backport from pydantic._internal._typing_extra, which pydantic 2.14 removed, so mcp.server.fastmcp fails to import with current pydantic (verified: mcp 1.20 imports with pydantic 2.13.5 and fails with 2.14.0; 1.22 is the first 1.x that imports with 2.14.0). The floor CI leg force-reinstalled mcp==1.20 with the newest pydantic and was red on main. The pin is now mcp>=1.22,<3.0 and the leg tests 1.22. --- .github/workflows/ci.yml | 10 +++++----- CHANGELOG.md | 6 ++++++ fluid_build/_mcp_compat.py | 6 +++--- pyproject.toml | 8 ++++++-- tests/test_mcp_sdk_rename_guard.py | 2 +- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1682e8f..aad9cf88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -408,12 +408,12 @@ jobs: strategy: fail-fast: false matrix: - # "1.20" = lower-bound pin from pyproject.toml `mcp>=1.20` + # "1.22" = lower-bound pin from pyproject.toml `mcp>=1.22` # (see comment there for the FastMCP API features we # need: deferred-annotation handling, ClientSession # ``sampling_capabilities`` kwarg, etc.) # "range-max" = newest release inside the pyproject constraint - # (`mcp>=1.20,<3.0`) — what a fresh + # (`mcp>=1.22,<3.0`) — what a fresh # `pip install data-product-forge` resolves today. # Since Phase 2 widened the ceiling this resolves to # 2.x, and this leg is NOT warn-wrapped, so 2.x is @@ -425,7 +425,7 @@ jobs: # warn-only, for the same reason it was warn-only # for 2.x: an upstream release must not be able to # paint every open PR red. - mcp-version: ["1.20", "range-max", "latest"] + mcp-version: ["1.22", "range-max", "latest"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -451,7 +451,7 @@ jobs: done case "${{ matrix.mcp-version }}" in # Keep the range-max constraint in sync with pyproject.toml. - range-max) pip install --upgrade "mcp>=1.20,<3.0" ;; + range-max) pip install --upgrade "mcp>=1.22,<3.0" ;; latest) pip install --upgrade mcp ;; *) pip install --force-reinstall "mcp==${{ matrix.mcp-version }}" ;; esac @@ -470,7 +470,7 @@ jobs: version = md.version("mcp") major = int(version.split(".")[0]) print(f"leg={leg} resolved mcp=={version} (major {major})") - expected = {"1.20": 1, "range-max": 2, "latest": 2}[leg] + expected = {"1.22": 1, "range-max": 2, "latest": 2}[leg] if major != expected: sys.exit( f"::error::leg '{leg}' resolved mcp {version}, expected major {expected}. " diff --git a/CHANGELOG.md b/CHANGELOG.md index ef30cb78..739c6a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Upgrade notes +- The minimum `mcp` is now 1.22 (was 1.20). mcp 1.20 and 1.21 import + `eval_type_backport` from pydantic's private `_typing_extra` module, which pydantic 2.14 + removed, so `mcp.server.fastmcp`, and every forge-cli MCP surface with it, failed to + import on a fresh install that resolved those releases with current pydantic. 1.22 is + the first 1.x that imports with pydantic 2.14. The CI drift leg that pins the floor moves + from 1.20 to 1.22. - An AWS contract that declared `location.catalog` other than `glue` (for example `rest`, `iceberg_rest`, `polaris`, `unity`, `nessie` or `lakekeeper`) on an Iceberg expose holds a Glue database and table in its OpenTofu state that earlier releases diff --git a/fluid_build/_mcp_compat.py b/fluid_build/_mcp_compat.py index 1648ff72..83423fed 100644 --- a/fluid_build/_mcp_compat.py +++ b/fluid_build/_mcp_compat.py @@ -35,9 +35,9 @@ ``open_inmemory_session``). This module is the ONLY place allowed to branch on the SDK generation. -Runtime code keeps the ``mcp>=1.20,<2.0`` install pin (pyproject.toml) -until the 2.x drift leg has a green track record — the dual support here -is what turns that leg green in the first place. +The install pin (pyproject.toml) is ``mcp>=1.22,<3.0``, so both +generations install — the dual support here is what keeps both drift legs +green. Tier-0 leaf rules: no module-scope ``mcp`` import (the startup budget in ``tests/perf/test_startup_budget.py`` forbids ``mcp`` on ``fluid --help``); diff --git a/pyproject.toml b/pyproject.toml index 3e454f8d..fa6dba50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,11 @@ dependencies = [ # the sampling capability via that kwarg so the floor must include it. # - The gateway depends on the lifespan API + ``mcp.shared.memory`` # test fixtures introduced in 1.20. - # 1.20+ has all of the above. Pinned by the mcp-sdk-drift CI job. + # - 1.20 and 1.21 import ``eval_type_backport`` from + # ``pydantic._internal._typing_extra``, which pydantic 2.14 removed, + # so ``mcp.server.fastmcp`` fails to import with current pydantic. + # 1.22 is the first 1.x that imports with pydantic 2.14. + # 1.22+ has all of the above. Pinned by the mcp-sdk-drift CI job. # Ceiling ``<3.0`` since Phase 2 (2026-09-14). The codebase is # DUAL-SUPPORT (1.x and 2.x) via the fluid_build/_mcp_compat.py seam, and # widening the ceiling is what makes 2.x BLOCKING: the drift job's @@ -142,7 +146,7 @@ dependencies = [ # AGENT_IDE.md "Related work" section + the gateway commit msg): # docs: https://modelcontextprotocol.github.io/python-sdk/ # repo: https://github.com/modelcontextprotocol/python-sdk - "mcp>=1.20,<3.0", + "mcp>=1.22,<3.0", # (The MCP output-port rate limiter is an in-process monotonic-clock # deque sliding window in SessionState.check_rate_limit — no dependency. # PyrateLimiter was dropped: its Limiter spins a background "leaker" diff --git a/tests/test_mcp_sdk_rename_guard.py b/tests/test_mcp_sdk_rename_guard.py index ac8f8bc7..9cb16ada 100644 --- a/tests/test_mcp_sdk_rename_guard.py +++ b/tests/test_mcp_sdk_rename_guard.py @@ -26,7 +26,7 @@ does not find the fields. Neither raises, so neither shows up as a test failure elsewhere. That is what this file is for. -The audit behind the ``mcp>=1.20,<3.0`` ceiling found zero of either. These +The audit behind the ``<3.0`` ceiling of ``mcp>=1.22,<3.0`` found zero of either. These tests pin that result so it cannot drift back unnoticed, in the same spirit as ``tests/perf/test_startup_budget.py``, which guards the cold-path import budget an ordinary test cannot express.