From 6eaa58d7dd5f56e23f35c13bf950d1e1d8ff3cf2 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 11:30:39 +0200 Subject: [PATCH 01/10] fix(platform): one contract on aws and gcp keeps two states, and GCP fails closed on sovereignty Platform safety for deploying one contract to two clouds through --env overlays, from the offline verification of the demo products on 0.16.5. State (F1). The default remote state key names the provider: fluid///terraform.tfstate (GCS prefix fluid//), for FLUID_STATE_BACKEND bucket-only specs and packaging contracts. The aws and the gcp apply of one contract shared fluid//terraform.tfstate, so each plan read the other cloud's resources as orphans to destroy. The first apply after the upgrade moves the old key's state with OpenTofu's own `tofu init -force-copy` (implies -migrate-state), only when the new key holds no state and the old one holds this provider's resources; another provider's state is left alone, a state of two clouds is refused, the old object is never deleted, and the copy is verified by reading it back. fluid diff / verify --state-drift read the old key while the move is pending. The shared legacy key and explicit keys are unchanged. Stage 8 (F2). info()/warn()/error() take logger and message positionally only, and a payload key named like an envelope key is kept as extra_: the GCP policy result's "message" raised a TypeError on every gcp build. Sovereignty (F5). Under enforcementMode strict a binding on aws, gcp or azure that names no region is refused (advisory warns, audit logs); a GCP region given as location.location is checked; BigQuery's US and EU multi-regions resolve to their jurisdictions. The GCP IaC plugin checks every emitted location and the GCP provider every planned one (the way AwsProvider.plan does), so fluid apply and fluid generate iac refuse an out-of-jurisdiction region, including a provider-default region a resource inherits. The BigQuery load runs where the table is, never a guessed US. Command Center (F10). fluid apply registers each run at POST /api/v1/executions and closes it at PATCH, through the existing CommandCenterReporter, with the credential and organization fluid publish uses: product id, contract version and hash, environment, provider, mode, state location, resource addresses, change counts, timings and status. Best effort: an outage or refusal never changes the exit code, and no secret, header or tofu output is sent. FLUID_COMMAND_CENTER_ENABLED=false turns it off. Small items (F11). An env's Airflow DAG id is ____ and its schedule directory __; every generated stage 6 runs plan --check-sovereignty; --env X with no overlay is an error when the contract's environments block or the workspace's expected-environments declares X for the product (dev and the base's own platform stay the base), and the warning otherwise names what the base binds to. Borrowed: OpenTofu backend migration (meta_backend_migrate.go), Terragrunt backend migrate, WebbPulse/webbpulse-python#129 (rename colliding log keys), the OpenLineage client's log-don't-fail posture, dbt's unknown-target error. --- .github/workflows/iac-tests.yml | 9 + .secrets.baseline | 4 +- fluid_build/build_runners/_bigquery_load.py | 15 +- fluid_build/cli/_apply_cc_report.py | 404 ++++++++++++++++++ fluid_build/cli/_apply_opentofu_engine.py | 174 +++++++- fluid_build/cli/_diff_state.py | 34 +- fluid_build/cli/_logging.py | 25 +- fluid_build/cli/apply.py | 3 + fluid_build/cli/bundle.py | 2 +- fluid_build/forge/core/artifact_fanout.py | 12 +- .../forge/core/pipeline_systems/_base.py | 13 +- .../forge/core/pipeline_systems/jenkins.py | 4 +- fluid_build/iac/backend.py | 79 +++- fluid_build/iac/providers/gcp.py | 11 + fluid_build/iac/runner.py | 33 +- fluid_build/iac/state_migration.py | 378 ++++++++++++++++ fluid_build/loader.py | 142 +++++- fluid_build/observability/config.py | 16 +- fluid_build/observability/reporter.py | 24 +- fluid_build/policy/sovereignty.py | 112 ++++- fluid_build/providers/gcp/provider.py | 32 ++ fluid_build/providers/gcp/util/sovereignty.py | 166 +++++++ fluid_build/schedulers/airflow/fluid_apply.py | 27 +- .../test_apply_reports_to_command_center.py | 376 ++++++++++++++++ tests/cli/test_diff_state_drift.py | 13 +- .../cli/test_generate_schedule_fluid_apply.py | 4 + .../test_one_contract_two_clouds_pipeline.py | 264 ++++++++++++ .../cli/test_policy_apply_provider_message.py | 139 ++++++ tests/forge/test_artifact_fanout_schedule.py | 11 +- .../iac/test_iac_state_key_migration_moto.py | 357 ++++++++++++++++ tests/iac/test_state_backend_env_default.py | 20 +- tests/iac/test_state_key_per_provider.py | 190 ++++++++ .../test_gcp_sovereignty_fail_closed.py | 333 +++++++++++++++ 33 files changed, 3345 insertions(+), 81 deletions(-) create mode 100644 fluid_build/cli/_apply_cc_report.py create mode 100644 fluid_build/iac/state_migration.py create mode 100644 fluid_build/providers/gcp/util/sovereignty.py create mode 100644 tests/cli/test_apply_reports_to_command_center.py create mode 100644 tests/cli/test_one_contract_two_clouds_pipeline.py create mode 100644 tests/cli/test_policy_apply_provider_message.py create mode 100644 tests/iac/test_iac_state_key_migration_moto.py create mode 100644 tests/iac/test_state_key_per_provider.py create mode 100644 tests/providers/test_gcp_sovereignty_fail_closed.py diff --git a/.github/workflows/iac-tests.yml b/.github/workflows/iac-tests.yml index 672813bd..8b216f3e 100644 --- a/.github/workflows/iac-tests.yml +++ b/.github/workflows/iac-tests.yml @@ -132,6 +132,15 @@ jobs: -q -p no:randomly --junitxml=state-drift.xml .venv/bin/python scripts/ci/assert_lane_coverage.py state-drift.xml \ --require "state drift=tests/iac/test_iac_state_drift_moto.py" + # The default state key names the provider, and the first apply after the + # upgrade moves the old key's state with `tofu init -migrate-state`. Only a + # real apply, move and plan prove the plan after it changes nothing. + - name: Stage 1 — per-provider state key and its migration (tofu vs moto, creds-free) + run: | + .venv/bin/python -m pytest tests/iac/test_iac_state_key_migration_moto.py \ + -q -p no:randomly --junitxml=state-key-migration.xml + .venv/bin/python scripts/ci/assert_lane_coverage.py state-key-migration.xml \ + --require "state key migration=tests/iac/test_iac_state_key_migration_moto.py" # ------------------------------------------------------------------- # Stage 2 — docker emulators. PR + push. diff --git a/.secrets.baseline b/.secrets.baseline index 9242602c..bc767575 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": ".github/workflows/iac-tests.yml", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 217 + "line_number": 226 } ], ".github/workflows/integration.yml": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-09-27T20:32:09Z" + "generated_at": "2026-09-28T09:29:35Z" } diff --git a/fluid_build/build_runners/_bigquery_load.py b/fluid_build/build_runners/_bigquery_load.py index 4359bdb7..88fb2595 100644 --- a/fluid_build/build_runners/_bigquery_load.py +++ b/fluid_build/build_runners/_bigquery_load.py @@ -69,7 +69,11 @@ def bigquery_load_target( """The table a binding loads into, or None when it is not a BigQuery table. Resolved with the IaC's own helpers, so the load names the dataset, table - and location ``_emit_bigquery`` created. + and location ``_emit_bigquery`` created. A binding that names no region + gives ``location: None``, never a guessed ``US``: :func:`load_file` then + runs the job where the table itself is (its ``location``), which is the + only place a load job can run. The guessed default could send a job for + an EU table to the US multi-region. """ from ..iac.providers.gcp import BIGQUERY_TABLE, _bq_table_name, resolve_gcp_target @@ -83,7 +87,7 @@ def bigquery_load_target( "project": project, "dataset": loc.get("dataset") or "default", "table": _bq_table_name(expose, loc), - "location": loc.get("region") or loc.get("location") or "US", + "location": loc.get("region") or loc.get("location") or None, } @@ -139,10 +143,11 @@ def load_file( create_disposition="CREATE_NEVER", schema=table.schema, ) + # The job runs where the table is: the binding's region when it names + # one, otherwise the table's own location, read above, never a default. + location = target.get("location") or getattr(table, "location", None) with open(path, "rb") as fh: - job = client.load_table_from_file( - fh, table_id, job_config=job_config, location=target["location"] - ) + job = client.load_table_from_file(fh, table_id, job_config=job_config, location=location) job.result() loaded = int(job.output_rows or 0) if loaded != expected_rows: diff --git a/fluid_build/cli/_apply_cc_report.py b/fluid_build/cli/_apply_cc_report.py new file mode 100644 index 00000000..be4a36ec --- /dev/null +++ b/fluid_build/cli/_apply_cc_report.py @@ -0,0 +1,404 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid apply`` reports each run to the Command Center, best effort. + +The Command Center records CLI runs at ``POST /api/v1/executions`` (a run +starts, ``status: running``) and ``PATCH /api/v1/executions/{id}`` (it ends: +``success`` / ``failed``, the Command Center sets ``completed_at`` and +``duration_seconds``). forge-cli has shipped a client for exactly that API +since the observability module landed (``observability/reporter.py``, +``CommandCenterReporter``: async queue, circuit breaker, SSRF host gate), but +nothing called it (``cli/bootstrap.py::get_reporter`` has no callers), so a +Jenkins run left no trace. This wires that client into ``fluid apply`` +rather than adding another one. + +**Where it reports, and as whom.** Wherever ``fluid publish`` already does: +the ``fluid-command-center`` catalog configuration (``FLUID_CC_ENDPOINT``, +``FLUID_API_KEY`` or ``FLUID_BEARER_TOKEN``, and the organization from +``FLUID_CC_ORG_ID``, ``organization_id`` or the ``organization`` slug in the +product's ``fluid.config.yaml``, resolved by the publisher's own +``resolve_organization_id``), so a pipeline configured to publish reports its +applies with no new setting. The reporter's own ``FLUID_COMMAND_CENTER_URL`` +/ ``FLUID_COMMAND_CENTER_API_KEY`` are the fallback. ``FLUID_COMMAND_CENTER_ENABLED=false`` +turns it off. Without an organization the run is not sent: the Command +Center would store it untagged, where no read path shows it. + +**What it says.** The run's product id, contract version and ``fluidVersion``, +the contract hash the Command Center keys contract versions by, the +``--env`` it was applied for, the provider, the apply mode, the state +location, the planned and applied change counts and the address of every +resource the module declares, and the timings. **Never a secret**: no header, +no environment value, no tofu output (its text can carry attribute values); +a failure is reported by its typed event name and exit code only. + +**Best effort.** A Command Center that is down, slow, refusing or +misconfigured costs the apply a warning line and at most the reporter's +timeout, never its exit code (the OpenLineage client's posture: failures +are logged, not raised). Every step below swallows its own errors. +""" + +from __future__ import annotations + +import asyncio +import contextvars +import functools +import logging +import os +import platform +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Mapping, Optional + +_LOG = logging.getLogger(__name__) + +#: The report of the ``fluid apply`` running in this context, if any. +_CURRENT: contextvars.ContextVar[Optional["ApplyRunReport"]] = contextvars.ContextVar( + "fluid_apply_cc_report", default=None +) + +_OFF_VALUES = {"0", "false", "no", "off"} + +#: Why nothing is reported when nothing is configured: silent, by design. +_NOT_CONFIGURED = "no Command Center is configured" + + +def current_report() -> Optional["ApplyRunReport"]: + """The report the running ``fluid apply`` fills, or ``None``.""" + return _CURRENT.get() + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _runner() -> str: + """The CC's ``runner`` tag: ``jenkins`` under Jenkins (which sets + ``JENKINS_URL`` for every build step), else ``cli``.""" + return "jenkins" if os.environ.get("JENKINS_URL") else "cli" + + +class ApplyRunReport: + """One ``fluid apply`` run, as the Command Center's executions API takes it.""" + + def __init__(self, args: Any, logger: logging.Logger) -> None: + self.logger = logger + self.execution_id = str(uuid.uuid4()) + self.started_at = _utc_now() + self._t0 = time.monotonic() + self.contract_path = str(getattr(args, "contract", "") or "") or None + self.environment: Optional[str] = getattr(args, "env", None) or None + self.mode = str(getattr(args, "mode", "") or "") or None + self.provider: Optional[str] = None + self.metadata: Dict[str, Any] = {} + self.result: Dict[str, Any] = {} + self._reporter: Any = None + self._disabled_reason: Optional[str] = None + self._began = False + self._finished = False + + # -- filled by the apply engine ------------------------------------ + + def begin( + self, + *, + contract: Mapping[str, Any], + provider: str, + environment: Optional[str] = None, + state: Optional[str] = None, + ) -> None: + """The engine knows the contract and the provider: register the run.""" + try: + self.provider = provider + if environment: + self.environment = environment + self.metadata.update(_contract_facts(contract)) + self.metadata["platform"] = provider + if state: + self.metadata["state"] = state + self._register() + except Exception as exc: # noqa: BLE001 - reporting never fails an apply + _LOG.debug("command center report: begin failed: %s", type(exc).__name__) + + def record_infra( + self, + *, + planned: Mapping[str, Any], + applied: Optional[Mapping[str, Any]], + resources: List[str], + dry_run: bool, + ) -> None: + """What the plan and the apply did, and which resources the module holds.""" + self.result.update( + { + "planned_changes": {k: int(planned.get(k, 0)) for k in ("add", "change", "remove")}, + "applied_changes": ( + None + if applied is None + else {k: int(applied.get(k, 0)) for k in ("add", "change", "remove")} + ), + "resources": list(resources), + "dry_run": bool(dry_run), + } + ) + + # -- the run's end ------------------------------------------------- + + def finish(self, status: str, *, event: Optional[str] = None, exit_code: int = 0) -> None: + """Close the run: ``success`` or ``failed``. Safe to call once, never raises.""" + if self._finished: + return + self._finished = True + try: + if not self._began: + self._register() + reporter = self._reporter + if reporter is None: + return + finished_at = _utc_now() + duration = round(time.monotonic() - self._t0, 3) + timings = { + "started_at": self.started_at, + "finished_at": finished_at, + "duration_seconds": duration, + } + result = dict(self.result, exit_code=int(exit_code), **timings) + if event: + result["error_event"] = str(event) + reporter.update_execution( + self.execution_id, + status=status, + progress=100.0, + current_phase="plan" if self.result.get("dry_run") else "apply", + error_message=(f"fluid apply failed: {event}" if event else None), + result=result, + ) + reporter.stop(timeout=float(reporter.config.timeout) * 2 + 1) + self._say_outcome(reporter) + except Exception as exc: # noqa: BLE001 - reporting never fails an apply + _LOG.debug("command center report: finish failed: %s", type(exc).__name__) + + # -- internals ----------------------------------------------------- + + def _register(self) -> None: + self._began = True + reporter = self._start_reporter() + if reporter is None: + return + from fluid_build import __version__ + + metadata = dict(self.metadata) + metadata.setdefault("environment", self.environment) + metadata["mode"] = self.mode + reporter.register_execution( + execution_id=self.execution_id, + command="apply", + contract_path=self.contract_path, + provider=self.provider, + environment=self.environment, + runner=_runner(), + cli_version=str(__version__), + python_version=platform.python_version(), + metadata=metadata, + ) + + def _start_reporter(self) -> Any: + if self._reporter is not None: + return self._reporter + config, reason = _command_center_config(self.logger) + if config is None: + self._disabled_reason = reason + if reason and reason != _NOT_CONFIGURED: + # Configured but unusable: say so once. Unconfigured is silent. + self._say(f" command center: run not reported ({reason})") + return None + from fluid_build.observability.reporter import CommandCenterReporter + + reporter = CommandCenterReporter(config) + reporter.start() + if not reporter.running: + # Refused by the reporter itself: its SSRF host gate, or no + # ``requests``. It has logged which. + self._disabled_reason = "the Command Center reporter did not start" + self._say(f" command center: run not reported ({self._disabled_reason})") + return None + self._reporter = reporter + return reporter + + def _say_outcome(self, reporter: Any) -> None: + sent, failed = reporter.stats.get("sent", 0), reporter.stats.get("failed", 0) + if failed or sent < 2: + self._say( + f" command center: run {self.execution_id} not fully reported " + f"({sent} of 2 requests accepted); the apply's result is unaffected" + ) + else: + self._say(f" command center: run {self.execution_id} reported") + + @staticmethod + def _say(line: str) -> None: + try: + from fluid_build.cli.console import cprint + + cprint(line) + except Exception: # noqa: BLE001 + pass + + +def _contract_facts(contract: Mapping[str, Any]) -> Dict[str, Any]: + """Identity facts only: nothing from the contract's body beyond its id and versions.""" + facts: Dict[str, Any] = { + "product_id": contract.get("id"), + "product_name": contract.get("name"), + "contract_version": contract.get("version"), + "fluid_version": contract.get("fluidVersion"), + } + try: + import yaml + + from fluid_build.providers.catalogs.fluid_cc.provider import command_center_contract_hash + + facts["contract_hash"] = command_center_contract_hash( + yaml.safe_dump(dict(contract), sort_keys=False) + ) + except Exception: # noqa: BLE001 - a hash is a join key, not required + pass + return facts + + +def _command_center_config(logger: logging.Logger): + """``(CommandCenterConfig, None)`` to report with, or ``(None, reason)``.""" + from fluid_build.observability.config import CommandCenterConfig + + enabled = os.environ.get("FLUID_COMMAND_CENTER_ENABLED") + if enabled is not None and enabled.strip().lower() in _OFF_VALUES: + return None, "FLUID_COMMAND_CENTER_ENABLED is off" + + published = _publish_path_config(logger) + if published is not None: + return published + env_config = CommandCenterConfig.from_environment() + if env_config.is_configured(): + org = (os.environ.get("FLUID_CC_ORG_ID") or "").strip() + if org and _header_safe(org): + env_config.headers = {**env_config.headers, "X-Organization-Id": org} + return env_config, None + return None, _NOT_CONFIGURED + + +def _publish_path_config(logger: logging.Logger): + """The ``fluid publish`` target, with its credential and its organization. + + ``None`` when the catalog configuration names no Command Center at all; + ``(None, reason)`` when it names one this run cannot report to. + """ + from fluid_build.config_manager import COMMAND_CENTER_CANONICAL_NAME, FluidConfig + from fluid_build.observability.config import CommandCenterConfig + + try: + catalog = FluidConfig().get_catalog_config(COMMAND_CENTER_CANONICAL_NAME) or {} + except Exception: # noqa: BLE001 - an unreadable config is "not configured" + return None + endpoint = str(catalog.get("endpoint") or "").strip() + if not endpoint or not catalog.get("enabled", True): + return None + # The built-in defaults name ``http://localhost:8000`` with no key, so an + # endpoint alone is not a configured Command Center: a credential is. + from fluid_build.providers.common import get_auth_headers + + probe = get_auth_headers(endpoint, catalog.get("auth")) + if not (probe.get("X-API-Key") or probe.get("Authorization")): + return None + # The reporter's SSRF gate, before the organization lookup below sends the + # credential anywhere: loopback or an allow-listed host, never a private + # or cloud-metadata address (observability/reporter.py). + from fluid_build.observability.reporter import _command_center_host_allowed + + if not _command_center_host_allowed(endpoint): + return None, ( + "its host resolves to a private or metadata address; allow it with " + "FLUID_COMMAND_CENTER_HOST_ALLOWLIST" + ) + try: + from fluid_build.providers.catalogs.fluid_cc import FluidCommandCenterProvider + + provider = FluidCommandCenterProvider(catalog) + org_id = _run_coroutine(provider.resolve_organization_id()) + headers = dict(provider._headers()) + except Exception as exc: # noqa: BLE001 - typed CC errors say what is missing + return None, f"the Command Center organization could not be settled ({type(exc).__name__})" + api_key = headers.pop("X-API-Key", None) + extra = {k: v for k, v in headers.items() if k in ("Authorization", "X-Organization-Id")} + if org_id and _header_safe(org_id): + extra["X-Organization-Id"] = org_id + timeout = _timeout(catalog.get("timeout")) + config = CommandCenterConfig( + url=provider.endpoint, api_key=api_key, timeout=timeout, headers=extra + ) + if not config.is_configured(): + return None, "the Command Center catalog configuration has no credential" + return config, None + + +def _timeout(value: Any) -> int: + try: + seconds = int(value) + except (TypeError, ValueError): + return 5 + return max(1, min(seconds, 10)) + + +def _header_safe(value: str) -> bool: + from fluid_build.providers.catalogs.fluid_cc.provider import _is_valid_org_id + + return bool(_is_valid_org_id(value)) + + +def _run_coroutine(coro: Any) -> Any: + """Run ``coro`` to completion from sync code, even under a running loop.""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + +def reports_apply_run(fn: Callable[..., int]) -> Callable[..., int]: + """Decorate ``fluid apply``'s ``run``: one Command Center run per invocation.""" + + @functools.wraps(fn) + def wrapper(args: Any, logger: logging.Logger) -> int: + from ._common import CLIError + + report = ApplyRunReport(args, logger) + token = _CURRENT.set(report) + try: + rc = fn(args, logger) + except CLIError as exc: + report.finish("failed", event=exc.event, exit_code=exc.exit_code) + raise + except BaseException as exc: + report.finish("failed", event=type(exc).__name__, exit_code=1) + raise + else: + report.finish("success" if rc == 0 else "failed", exit_code=rc) + return rc + finally: + _CURRENT.reset(token) + + return wrapper diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index 8c14a04b..48834d05 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -29,7 +29,7 @@ import json import logging from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple @@ -38,13 +38,22 @@ from fluid_build.iac.backend import ( STATE_BACKEND_ENV, backend_location, + legacy_default_backend, parse_backend, resolve_state_backend_spec, ) from fluid_build.iac.base import UnsupportedBindingError from fluid_build.iac.credentials import build_tofu_env, credential_report from fluid_build.iac.naming import safe_ident +from fluid_build.iac.state_migration import ( + PENDING, + StateMigrationError, + StateReconciliation, + records_backend, +) +from fluid_build.iac.state_migration import reconcile_state_key as _reconcile_state +from ._apply_cc_report import current_report from ._common import CLIError, load_contract_with_overlay, resolve_env_templates_in_contract from ._logging import info, warn from .generate_iac import _resolve_provider, native_actions @@ -129,10 +138,33 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: cprint(f" state: {state_line}") cprint(f" credentials: {', '.join(present) if present else 'none detected in environment'}") - init = runner.tofu_init(str(workdir), backend=backend is not None, env=env) + # The Command Center hears about the run now that the product, the + # provider and the environment are known (best effort; see + # cli/_apply_cc_report.py). Addresses and counts only, never tofu output. + report = current_report() + if report is not None: + report.begin( + contract=contract, + provider=provider, + environment=_applied_env(args), + state=target.location, + ) + + init = runner.tofu_init( + str(workdir), + backend=backend is not None, + env=env, + reconfigure=recorded_legacy_backend(target), + ) if not init.ok: raise CLIError(1, "opentofu_init_failed", {"error": _tail(init.stderr or init.stdout)}) + # State a previous release kept at the key without the provider moves to + # this provider's key (OpenTofu's own ``init -migrate-state``), so the + # first apply after the upgrade does not plan every resource as new + # (see ``iac.state_migration``). Before anything reads the state. + reconcile_state_key(target, provider, env, logger, migrate=True) + # Pre-plan region guard. A module now pins the region its bindings name, # and moving a contract's resources to it would not show as a destroy. _guard_region_move(plugin, contract, str(workdir), env) @@ -153,6 +185,13 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: raise CLIError(1, "opentofu_plan_failed", {"error": _tail(plan.stderr or plan.stdout)}) changes = runner.change_summary(plan) cprint(f"\n tofu plan: +{changes['add']} ~{changes['change']} -{changes['remove']}") + if report is not None: + report.record_infra( + planned=changes, + applied=None, + resources=_module_addresses(module), + dry_run=bool(getattr(args, "dry_run", False)), + ) # Report what the plan's ``lifecycle.ignore_changes`` deliberately hides. # Without this, a contract whose column types no longer match the live @@ -225,11 +264,45 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: record(applied) cprint(f"\n tofu apply complete: +{applied['add']} ~{applied['change']} -{applied['remove']}") + if report is not None: + report.record_infra( + planned=changes, applied=applied, resources=_module_addresses(module), dry_run=False + ) info(logger, "opentofu_apply_ok", provider=provider, **applied) return 0 +def _applied_env(args) -> Optional[str]: + """The overlay env this apply is for: ``--env``, else the bundle's own.""" + env = getattr(args, "env", None) + if env: + return str(env) + bundle = getattr(args, "bundle", None) + if not bundle: + return None + try: + from fluid_build.forge.core.bundle import read_bundle_source + + source = read_bundle_source(Path(bundle)) or {} + except Exception: # noqa: BLE001 - the env is a report field, not a gate + return None + return str(source["env"]) if source.get("env") else None + + +def _module_addresses(module_text: str) -> List[str]: + """``type.name`` of every resource the emitted module declares.""" + try: + doc = json.loads(module_text) + except ValueError: + return [] + out: List[str] = [] + for rtype, by_name in sorted((doc.get("resource") or {}).items()): + if isinstance(by_name, dict): + out.extend(f"{rtype}.{name}" for name in sorted(by_name)) + return out + + @dataclass(frozen=True) class StateTarget: """Where ``fluid apply`` keeps one contract's OpenTofu workdir and state.""" @@ -239,6 +312,9 @@ class StateTarget: backend: Optional[Dict[str, Any]] #: ``--state-backend``, ``FLUID_STATE_BACKEND`` or ``default``. origin: str + #: Where a release before the provider-keyed default kept this state, when + #: that differs from ``backend`` (``iac.backend.legacy_default_backend``). + legacy_backend: Optional[Dict[str, Any]] = None @property def location(self) -> str: @@ -269,12 +345,21 @@ def resolve_state_target(args, contract: Mapping[str, Any], provider: str) -> St # bucket instead of the workspace it wipes after every run. That job # applies every product with the one value, so a bucket-only value keys # state per contract for all of them, packaging block or not. + # + # The provider is part of every per-contract default key, so one contract + # applied to aws and to gcp (``--env`` overlays) keeps two states: with + # one, each cloud's plan read the other's resources as orphans to destroy. backend_spec, backend_origin = resolve_state_backend_spec(getattr(args, "state_backend", None)) + per_contract = backend_origin == STATE_BACKEND_ENV try: backend = parse_backend( backend_spec, contract, - per_contract_default=backend_origin == STATE_BACKEND_ENV, + per_contract_default=per_contract, + provider=provider, + ) + legacy = legacy_default_backend( + backend_spec, contract, per_contract_default=per_contract, provider=provider ) except ValueError as exc: raise CLIError( @@ -292,7 +377,88 @@ def resolve_state_target(args, contract: Mapping[str, Any], provider: str) -> St / provider / safe_ident(contract.get("id") or "contract") ) - return StateTarget(workdir=workdir, backend=backend, origin=backend_origin) + return StateTarget( + workdir=workdir, backend=backend, origin=backend_origin, legacy_backend=legacy + ) + + +def recorded_legacy_backend(target: StateTarget) -> bool: + """True when the workdir's ``.terraform/`` recorded the pre-provider-key backend. + + A plain ``tofu init`` stops there with "Backend configuration changed"; + the caller inits with ``-reconfigure`` instead, and :func:`reconcile_state_key` + moves the state, so nothing is left behind. + """ + return target.legacy_backend is not None and records_backend( + target.workdir, target.legacy_backend + ) + + +def reconcile_state_key( + target: StateTarget, + provider: str, + env: Mapping[str, str], + logger: logging.Logger, + *, + migrate: bool, +) -> Optional[StateReconciliation]: + """Bring state a previous release kept without the provider in its key along. + + Runs after the workdir's ``tofu init`` on ``target.backend``. + ``migrate=True`` (``fluid apply``) copies it with ``tofu init + -migrate-state``; ``migrate=False`` (the read-only drift pass) only + reports it, and :func:`read_target` then points the read at the old key. + ``None`` when there is no old key to look at. + """ + if target.backend is None or target.legacy_backend is None: + return None + try: + outcome = _reconcile_state( + workdir=target.workdir, + current=target.backend, + legacy=target.legacy_backend, + provider=provider, + env=env, + migrate=migrate, + logger=logger, + ) + except StateMigrationError as exc: + raise CLIError( + 1, + exc.code, + { + "error": str(exc), + "state": backend_location(target.backend), + "legacy_state": backend_location(target.legacy_backend), + }, + ) + line = outcome.summary() + if line: + cprint(f" state move: {line}") + info( + logger, + "opentofu_state_key_reconciled", + outcome=outcome.outcome, + state=backend_location(target.backend), + legacy_state=backend_location(target.legacy_backend), + resources=outcome.resources, + ) + return outcome + + +def read_target( + target: StateTarget, provider: str, env: Mapping[str, str], logger: logging.Logger +) -> StateTarget: + """The target a read-only caller reads: the old key while its move is pending. + + Call after the workdir's init on ``target.backend``. When this returns a + different target, the caller re-emits the module on its backend and + re-inits with ``-reconfigure``. + """ + outcome = reconcile_state_key(target, provider, env, logger, migrate=False) + if outcome is None or outcome.outcome != PENDING: + return target + return replace(target, backend=dict(outcome.legacy)) def emit_module( diff --git a/fluid_build/cli/_diff_state.py b/fluid_build/cli/_diff_state.py index 2fdef8ca..e9eef65c 100644 --- a/fluid_build/cli/_diff_state.py +++ b/fluid_build/cli/_diff_state.py @@ -184,7 +184,13 @@ def _plan_and_classify( from fluid_build.iac import runner from fluid_build.iac.credentials import build_tofu_env - from ._apply_opentofu_engine import _guard_region_move, _tail, emit_module + from ._apply_opentofu_engine import ( + _guard_region_move, + _tail, + emit_module, + read_target, + recorded_legacy_backend, + ) workdir: Path = target.workdir location = target.location @@ -200,13 +206,37 @@ def _plan_and_classify( env = build_tofu_env() env.update(plugin.credential_env(env)) - init = runner.tofu_init(str(workdir), backend=target.backend is not None, env=env) + init = runner.tofu_init( + str(workdir), + backend=target.backend is not None, + env=env, + reconfigure=recorded_legacy_backend(target), + ) if not init.ok: return _error( "OpenTofu could not initialise the apply's workdir: " + _tail(init.stderr or init.stdout, _MAX_DETAIL_CHARS), location, ) + # State a previous release kept at the key without the provider is + # read where it is until ``fluid apply`` moves it: this pass writes + # no state, and a drift gate that runs before the first upgraded + # apply must still see the real one. + try: + read = read_target(target, plugin.name, env, logger) + except CLIError as exc: + return _error(_cli_error_text(exc), location) + if read is not target: + target, location = read, read.location + module, _actions = emit_module(plugin, contract, target, logger) + module_path.write_text(module, encoding="utf-8") + init = runner.tofu_init(str(workdir), backend=True, env=env, reconfigure=True) + if not init.ok: + return _error( + "OpenTofu could not initialise the apply's workdir on the old state key: " + + _tail(init.stderr or init.stdout, _MAX_DETAIL_CHARS), + location, + ) # The apply refuses to run when state holds the contract's resources in # another region; the refresh would find them gone and this pass would # call that "deleted outside the apply". diff --git a/fluid_build/cli/_logging.py b/fluid_build/cli/_logging.py index 0e91c45d..72eafe24 100644 --- a/fluid_build/cli/_logging.py +++ b/fluid_build/cli/_logging.py @@ -38,19 +38,36 @@ def setup_logging(level: str = "INFO", file: str | None = None) -> logging.Logge return logger +#: The envelope keys every event carries. A payload key of the same name is +#: kept as ``extra_`` instead of replacing the envelope's (the event +#: name is ``message``; a provider result with its own ``message`` would +#: otherwise have renamed the event). +_ENVELOPE_KEYS = frozenset({"time", "level", "name", "message"}) + + def _event(level: str, name: str, payload: Dict[str, Any]) -> str: + # Renamed, not dropped: the pattern of WebbPulse/webbpulse-python#129 + # (``extra_`` for an ``extra`` key that collides with a LogRecord + # attribute). structlog's hynek/structlog#842 drops such keys instead, + # which would lose the provider's own explanation here. + fields = {(f"extra_{k}" if k in _ENVELOPE_KEYS else k): v for k, v in payload.items()} return json.dumps( { "time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "name": "fluid.cli", "message": name, - **payload, + **fields, } ) -def info(logger: logging.Logger, message: str, **payload: Any) -> None: +# ``logger`` and ``message`` are positional-only (PEP 570), so a payload may +# carry keys of those names: ``info(logger, "policy_apply_result", **res)`` +# raised ``TypeError: info() got multiple values for argument 'message'`` +# for every provider whose result has a ``message`` (GCP's policy applier +# does), and failed stage 8 of every generated gcp pipeline. +def info(logger: logging.Logger, message: str, /, **payload: Any) -> None: """Emit a structured INFO event to the log sink. Routed at DEBUG level for the human-facing console handler so the @@ -67,7 +84,7 @@ def info(logger: logging.Logger, message: str, **payload: Any) -> None: logger.debug(_event("INFO", message, payload)) -def warn(logger: logging.Logger, message: str, **payload: Any) -> None: +def warn(logger: logging.Logger, message: str, /, **payload: Any) -> None: """Emit a structured WARNING event — stays at WARNING level. Warnings are user-relevant ("we didn't break, but you should know @@ -76,5 +93,5 @@ def warn(logger: logging.Logger, message: str, **payload: Any) -> None: logger.warning(_event("WARNING", message, payload)) -def error(logger: logging.Logger, message: str, **payload: Any) -> None: +def error(logger: logging.Logger, message: str, /, **payload: Any) -> None: logger.error(_event("ERROR", message, payload)) diff --git a/fluid_build/cli/apply.py b/fluid_build/cli/apply.py index 2924c0ff..197f6539 100644 --- a/fluid_build/cli/apply.py +++ b/fluid_build/cli/apply.py @@ -46,6 +46,8 @@ from fluid_build.cli.console import cprint, success, warning from fluid_build.observability.tracing import traced_stage as _traced_stage +from ._apply_cc_report import reports_apply_run + # Rich imports for enhanced output try: from rich.console import Console @@ -1553,6 +1555,7 @@ def run_in_thread(): @_traced_stage("apply") +@reports_apply_run def run(args, logger: logging.Logger) -> int: """ Main execution function for the apply command diff --git a/fluid_build/cli/bundle.py b/fluid_build/cli/bundle.py index 6fd0e0d6..14138de8 100644 --- a/fluid_build/cli/bundle.py +++ b/fluid_build/cli/bundle.py @@ -228,7 +228,7 @@ def materialize_contract( compiled = _deep_merge(dict(compiled), overlay) logger.info("overlay_applied", extra={"overlay": str(overlay_path)}) else: - note_missing_overlay(contract_path, env, logger) + note_missing_overlay(contract_path, env, logger, contract=compiled) return compiled, overlay_path diff --git a/fluid_build/forge/core/artifact_fanout.py b/fluid_build/forge/core/artifact_fanout.py index a3e129ba..7def0a79 100644 --- a/fluid_build/forge/core/artifact_fanout.py +++ b/fluid_build/forge/core/artifact_fanout.py @@ -32,7 +32,7 @@ ├── odcs/product.odcs..yaml # ODCS v3.1.0 (bitol-io) — one per exposed port ├── odps-bitol/.odps.yaml # ODPS-Bitol v1.0.0 (bitol-io) ├── opds/.opds.json # OPDS v4.1 (LF/ODPI) — schema-validated - ├── schedule// # one directory per product, so + ├── schedule/[__]/ # one directory per product, so │ └── _dag.py # schedule-sync never deletes │ # another product's DAGs (Path A) └── policy/bindings.json # compiled IAM/GRANT bindings @@ -558,7 +558,7 @@ def _emit_schedule( dag_contract_path: Optional[str] = None, warn_no_env: bool = False, ) -> List[Path]: - """DAG/flow emission via ``generate schedule`` into ``/schedule//``. + """DAG/flow emission via ``generate schedule`` into ``/schedule/[__]/``. ``env`` is the ``--env`` a ``fluid apply`` DAG passes on every run. ``overlay_env`` is the overlay applied while rendering: the same env for a @@ -608,11 +608,13 @@ def _emit_schedule( ) dag_contract_path = fluid_apply.DEFAULT_CONTRACT_PATH - # One directory per product: stage 11 (``schedule-sync``, default + # One directory per product and env: stage 11 (``schedule-sync``, default # ``--delete-scope product``) mirrors it into the same-named directory of # the scheduler's DAG root, so deleting stale DAGs never reaches another - # product's files there. - scope_dir = out_dir / product_id + # product's files there, nor the same product's DAGs for another env (an + # aws and a gcp pipeline syncing to one Airflow). ```` with no + # env, ``__`` with one. + scope_dir = out_dir / fluid_apply.schedule_scope_for(product_id, env or None) scope_dir.mkdir(parents=True, exist_ok=True) args = argparse.Namespace( contract=str(contract_path), diff --git a/fluid_build/forge/core/pipeline_systems/_base.py b/fluid_build/forge/core/pipeline_systems/_base.py index 24967322..b65f2fa2 100644 --- a/fluid_build/forge/core/pipeline_systems/_base.py +++ b/fluid_build/forge/core/pipeline_systems/_base.py @@ -520,7 +520,14 @@ def _get_fluid_commands(self, config: Optional["PipelineConfig"] = None) -> Dict "fluid diff ${CONTRACT:-contract.fluid.yaml} --exit-on-drift " "--env ${FLUID_ENV:-dev}" ), - "plan": "fluid plan ${CONTRACT:-contract.fluid.yaml} --out runtime/plan.json", + # --check-sovereignty: the plan stage runs the contract's own + # sovereignty policy (the provider hook, else the policy engine) + # and a strict violation fails it, before stage 7 touches a cloud. + # A contract with no sovereignty block prints NOT CHECKED and passes. + "plan": ( + "fluid plan ${CONTRACT:-contract.fluid.yaml} --out runtime/plan.json " + "--check-sovereignty" + ), # A build id needs `--mode amend-and-build` alongside # `--build-id`: the id only FILTERS, it does not opt into running # builds (`fluid apply --help`). The retired `--build` did both. @@ -1109,10 +1116,12 @@ def _stage_specs(self, config: Optional["PipelineConfig"] = None) -> List["Stage # The plan records the mode it was made for, and stage 7's # ``apply_plan_mode_mismatch`` gate refuses any other. So # stage 6 plans for APPLY_MODE, the one mode stage 7 applies. + # --check-sovereignty makes a strict sovereignty violation fail + # the plan stage, before stage 7 reaches a cloud. command=( f'set -eu; {_needs_bundle(6)}MODE="{p("APPLY_MODE")}"; ' f"fluid plan {BUNDLE_PATH} " - f'--out runtime/plan.json --mode "$MODE" {env}' + f'--out runtime/plan.json --mode "$MODE" {env} --check-sovereignty' ), ), StageSpec( diff --git a/fluid_build/forge/core/pipeline_systems/jenkins.py b/fluid_build/forge/core/pipeline_systems/jenkins.py index e4c58fa0..6f010549 100644 --- a/fluid_build/forge/core/pipeline_systems/jenkins.py +++ b/fluid_build/forge/core/pipeline_systems/jenkins.py @@ -718,9 +718,11 @@ def when(num: int, extra: str = "") -> str: [ "mkdir -p runtime", _needs_bundle(6), + # --check-sovereignty: a strict sovereignty violation fails the + # plan stage, before stage 7 reaches a cloud. ( f"set -- {BUNDLE_PATH} {env_flag} " - f'--mode "{v("APPLY_MODE")}" --out runtime/plan.json' + f'--mode "{v("APPLY_MODE")}" --out runtime/plan.json --check-sovereignty' ), ( f'if [ "{v("PLAN_HTML")}" = "true" ]; then ' diff --git a/fluid_build/iac/backend.py b/fluid_build/iac/backend.py index 0246b051..f16af35c 100644 --- a/fluid_build/iac/backend.py +++ b/fluid_build/iac/backend.py @@ -39,6 +39,18 @@ ``_``, so ``a.b``, ``a-b`` and ``a_b``, all valid ids, would share one state again. An id outside the FLUID identifier grammar cannot key a state and is refused. The variable is new, so no state lives at a key it chose before. + +**The provider is part of the default key.** One contract deployed to two +clouds through overlays (``--env aws`` and ``--env gcp``) is one id, so a key +made of the id alone put the aws and the gcp apply in one state: each plan +then read the other cloud's resources as orphans to destroy, and +``--allow-data-loss`` would have destroyed them. Every per-contract default is +now ``fluid///terraform.tfstate`` (the GCS prefix +``fluid//``) when the caller names the provider, as ``fluid +apply`` always does. The shared legacy key ``fluid/terraform.tfstate`` is +unchanged, and so is an explicit key in the spec. State the previous default +wrote is moved by :mod:`fluid_build.iac.state_migration`, with OpenTofu's own +``init -migrate-state``; :func:`legacy_default_backend` names where it was. """ from __future__ import annotations @@ -68,6 +80,11 @@ #: prints on its state line (a newline there would forge a line of output). _CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]") +#: What a provider name in a state key may hold: the IaC plugin names +#: (``aws``, ``gcp``, ``snowflake``, ``confluent``) and nothing that could add +#: a path segment or a control character to the key. +_PROVIDER_RE = re.compile(r"[a-z][a-z0-9_-]{0,31}") + #: Environment variable ``fluid apply`` reads for the state backend when #: ``--state-backend`` is not on the command line. A CI job sets it once so #: OpenTofu state lives in a bucket rather than the workspace, which CI @@ -101,13 +118,24 @@ def resolve_state_backend_spec( return (None, "default") -def default_state_key(contract: Optional[Mapping[str, Any]], *, per_contract: bool = False) -> str: +def default_state_key( + contract: Optional[Mapping[str, Any]], + *, + per_contract: bool = False, + provider: Optional[str] = None, +) -> str: """The default state key for ``contract`` — legacy unless packaging is declared. Returns :data:`LEGACY_STATE_KEY` when ``contract`` is ``None`` or resolves to the ``packaging.LEGACY`` sentinel, and the per-contract ``fluid//terraform.tfstate`` otherwise. + ``provider`` (``fluid apply`` passes the one it resolved) adds a segment to + every per-contract key, ``fluid///terraform.tfstate``, so the + same contract applied to two clouds keeps two states (see the module + docstring). The legacy shared key never takes it. A provider name outside + ``[a-z][a-z0-9_-]*`` is a ``ValueError``. + ``per_contract=True`` (the spec came from :data:`STATE_BACKEND_ENV`; see the module docstring) skips the packaging test and keys every contract by its id as written, ``fluid//terraform.tfstate``, so two distinct ids @@ -123,8 +151,9 @@ def default_state_key(contract: Optional[Mapping[str, Any]], *, per_contract: bo """ if contract is None: return LEGACY_STATE_KEY + segment = "" if provider is None else f"{_state_provider(provider)}/" if per_contract: - return f"fluid/{_state_id(contract)}/terraform.tfstate" + return f"fluid/{_state_id(contract)}/{segment}terraform.tfstate" try: resolution = resolve_packaging(contract) except PackagingError: @@ -132,7 +161,17 @@ def default_state_key(contract: Optional[Mapping[str, Any]], *, per_contract: bo if resolution is LEGACY: return LEGACY_STATE_KEY cid = safe_ident(contract.get("id") or contract.get("name") or "contract") - return f"fluid/{cid}/terraform.tfstate" + return f"fluid/{cid}/{segment}terraform.tfstate" + + +def _state_provider(provider: str) -> str: + """``provider``, once it is proven to be one safe key segment.""" + if isinstance(provider, str) and _PROVIDER_RE.fullmatch(provider): + return provider + raise ValueError( + f"provider {provider!r} cannot name a state key segment " + "([a-z][a-z0-9_-]*, at most 32 characters)" + ) def _state_id(contract: Mapping[str, Any]) -> str: @@ -153,6 +192,7 @@ def parse_backend( contract: Optional[Mapping[str, Any]] = None, *, per_contract_default: bool = False, + provider: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """Parse a backend spec into a ``terraform.backend`` block. @@ -167,6 +207,8 @@ def parse_backend( contracts, the shared legacy key otherwise. ``per_contract_default`` gives every contract its own default, keyed by its id as written (``fluid apply`` sets it for a spec from :data:`STATE_BACKEND_ENV`). + ``provider`` puts the provider into every per-contract default (see + :func:`default_state_key`); an explicit key or prefix is never changed. The backend block carries no credentials — ``tofu`` reads those from the environment (``AWS_*`` / ``GOOGLE_*``). A bucket name holding @@ -184,7 +226,7 @@ def parse_backend( _check_bucket(bucket, "s3") _check_path(key, "s3", "key") if not key: - key = default_state_key(contract, per_contract=per_contract_default) + key = default_state_key(contract, per_contract=per_contract_default, provider=provider) return {"s3": {"bucket": bucket, "key": key}} if spec.startswith("gcs://"): @@ -199,7 +241,9 @@ def parse_backend( # derive it from the same per-contract default (sans filename) so # both backends isolate identically. Legacy contracts emit no # prefix at all, exactly as before. - default = default_state_key(contract, per_contract=per_contract_default) + default = default_state_key( + contract, per_contract=per_contract_default, provider=provider + ) prefix = default.rsplit("/", 1)[0] if default != LEGACY_STATE_KEY else "" if prefix: block["gcs"]["prefix"] = prefix @@ -212,6 +256,31 @@ def parse_backend( raise ValueError(f"unsupported state backend {named} — use s3:// or gcs://") +def legacy_default_backend( + spec: Optional[str], + contract: Optional[Mapping[str, Any]], + *, + per_contract_default: bool, + provider: str, +) -> Optional[Dict[str, Any]]: + """Where the default state was before the provider joined the key, or None. + + The block :func:`parse_backend` returned for the same spec and contract + without ``provider``: ``fluid//terraform.tfstate`` (GCS prefix + ``fluid/``). ``None`` when there is nothing to migrate from: local + state, an explicit key or prefix (never changed, so never moved), or a + default that did not change (the shared legacy key). Raises + ``ValueError`` exactly where :func:`parse_backend` does. + """ + legacy = parse_backend(spec, contract, per_contract_default=per_contract_default) + current = parse_backend( + spec, contract, per_contract_default=per_contract_default, provider=provider + ) + if legacy is None or current is None or legacy == current: + return None + return legacy + + def _check_bucket(bucket: str, scheme: str) -> None: if not _BUCKET_RE.fullmatch(bucket): # Not echoed: what is not a bucket name may be a credential. diff --git a/fluid_build/iac/providers/gcp.py b/fluid_build/iac/providers/gcp.py index 4a0ef302..1c156735 100644 --- a/fluid_build/iac/providers/gcp.py +++ b/fluid_build/iac/providers/gcp.py @@ -388,6 +388,17 @@ def emit( # planner already interpreted the loose `execution.trigger` # surface into structured `run.*` / `scheduler.*` / `ps.*` ops. _emit_from_actions(resources, actions, cid) + # The GCP sovereignty hook, where the data lands: every location an + # emitted resource carries (a region the binding left to a default + # included) and each gcp expose that names no region. A refusal is + # raised before any module exists, for `fluid apply` and `fluid + # generate iac` alike (providers/gcp/util/sovereignty.py). + from ...providers.gcp.util.sovereignty import ( + enforce_gcp_sovereignty, + resource_placements, + ) + + enforce_gcp_sovereignty(contract, resource_placements(resources)) return resources def emit_data( diff --git a/fluid_build/iac/runner.py b/fluid_build/iac/runner.py index ab7da5a9..b24ff078 100644 --- a/fluid_build/iac/runner.py +++ b/fluid_build/iac/runner.py @@ -199,14 +199,43 @@ def _run( ) -def tofu_init(workdir: str, *, backend: bool = True, env: Optional[Mapping[str, str]] = None): - """``tofu init`` — install providers (and initialise the backend).""" +def tofu_init( + workdir: str, + *, + backend: bool = True, + env: Optional[Mapping[str, str]] = None, + reconfigure: bool = False, + force_copy: bool = False, +): + """``tofu init`` — install providers (and initialise the backend). + + ``reconfigure`` passes ``-reconfigure``: take the module's backend as + given and ignore the one ``.terraform/`` recorded, moving no state. + ``force_copy`` passes ``-force-copy``, which implies ``-migrate-state``: + copy the recorded backend's state into the module's backend without a + prompt, overwriting whatever the destination holds (OpenTofu + ``backendMigrateState_s_s``), so a caller checks the destination first. + """ args = ["init", "-input=false", "-no-color"] if not backend: args.append("-backend=false") + if reconfigure: + args.append("-reconfigure") + if force_copy: + args.append("-force-copy") return _run(args, workdir=workdir, env=env, command="init") +def tofu_state_pull(workdir: str, *, env: Optional[Mapping[str, str]] = None) -> TofuResult: + """``tofu state pull`` — the backend's current state document, as JSON text. + + A key that holds no state prints a document with an empty ``lineage`` + and serial 0 (measured on OpenTofu 1.12 against an S3 backend), not an + error; see :mod:`fluid_build.iac.state_migration` for how that is read. + """ + return _run(["state", "pull"], workdir=workdir, env=env, command="state-pull") + + def tofu_validate(workdir: str, *, env: Optional[Mapping[str, str]] = None) -> TofuResult: """``tofu validate`` — check config syntax + provider-schema correctness.""" return _run(["validate", "-no-color"], workdir=workdir, env=env, command="validate") diff --git a/fluid_build/iac/state_migration.py b/fluid_build/iac/state_migration.py new file mode 100644 index 00000000..d077d8a0 --- /dev/null +++ b/fluid_build/iac/state_migration.py @@ -0,0 +1,378 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Move a contract's state to its per-provider key, with OpenTofu's own migration. + +The default remote state key gained the provider +(``fluid///terraform.tfstate``, see :mod:`.backend`), so a +contract applied to aws and to gcp keeps two states instead of one that each +cloud's plan would read as the other's orphans. State a previous release +wrote at ``fluid//terraform.tfstate`` has to follow, or the first apply +after the upgrade plans every resource as new. + +**The mechanism is OpenTofu's.** Nothing here reads, edits or writes a state +document's content. The copy is ``tofu init -force-copy`` (which implies +``-migrate-state``) in a scratch directory whose ``.terraform/`` recorded the +old backend and whose module names the new one: the backend-reconfiguration +path ``terraform init -migrate-state`` has always had, which locks both +states where the backend supports locking and leaves the source untouched +(OpenTofu ``internal/command/meta_backend_migrate.go``, +``backendMigrateState_s_s``). The copy is checked afterwards by reading it +back: the same resources, since OpenTofu gives a copy into an empty +destination a fresh lineage. Terragrunt's ``backend migrate`` wraps the same +step for a renamed unit; this is that idea without the wrapper. The scratch +directory holds no provider, so the step downloads nothing. + +**Never lose it, never guess.** ``-force-copy`` overwrites a destination that +holds state (the same OpenTofu function skips its confirmation), so the copy +runs only when the new key holds none, checked once before deciding and again +just before copying. The old object is left where it was. Which provider a +state belongs to is read from its resources' own ``provider`` addresses +(``provider["registry.opentofu.org/hashicorp/aws"]``) against the provider +sources each IaC plugin requires: + +* every resource under this provider's plugin → migrate it; +* every resource under exactly one other plugin (the gcp apply finding the + aws state at the old key) → leave it, it is not this apply's; +* anything else (both clouds in one state, a provider no plugin emits, only + shared utility providers) → refuse with a typed error naming both keys, so + an operator decides. Moving the wrong state is the one mistake that cannot + be undone by the next apply. + +A read-only caller (``fluid diff``, ``fluid verify --state-drift``) asks with +``migrate=False`` and reads the old key while the move is pending, so a drift +gate that runs before the first upgraded apply still sees the real state. +""" + +from __future__ import annotations + +import json +import logging +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, FrozenSet, Iterable, Mapping, Optional, Tuple + +from . import runner +from .backend import backend_location + +#: The new key already holds state: it is used and nothing moves. +CURRENT = "current" +#: Neither key holds a state with resources in it. +NOTHING = "nothing" +#: The old key holds another provider's state, which is left alone. +OTHER_PROVIDER = "other_provider" +#: The old key holds this provider's state and the caller asked not to move it. +PENDING = "pending" +#: The old key's state was copied to the new key. +MIGRATED = "migrated" + +#: Provider sources a previous release wrote under a name the pins no longer +#: use: the Snowflake provider moved from ``Snowflake-Labs`` to ``snowflakedb`` +#: at v2 (see ``versions.PROVIDER_PINS``). +_SOURCE_ALIASES: Dict[str, Tuple[str, ...]] = { + "snowflake": ("snowflake-labs/snowflake",), +} + +#: ``provider["registry.opentofu.org/hashicorp/aws"]`` (optionally ``.alias``). +_PROVIDER_ADDR_RE = re.compile(r'provider\["([^"]+)"\]') + +_LOG = logging.getLogger(__name__) + + +class StateMigrationError(RuntimeError): + """The state could not be moved, or whose it is could not be told.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + + +@dataclass(frozen=True) +class StateDoc: + """What ``tofu state pull`` said is at one key.""" + + lineage: str + serial: int + resources: Tuple[Mapping[str, Any], ...] + + @property + def exists(self) -> bool: + """A key with no state pulls as an empty lineage (measured, tofu 1.12).""" + return bool(self.lineage) + + +@dataclass(frozen=True) +class StateReconciliation: + """What :func:`reconcile_state_key` found and did.""" + + outcome: str + #: Where the state was before the provider joined the key. + legacy: Mapping[str, Any] + #: Where it is now (the key the apply uses). + current: Mapping[str, Any] + resources: int = 0 + detail: str = "" + + @property + def read_from(self) -> Mapping[str, Any]: + """The backend a read-only caller should read: the old one while pending.""" + return self.legacy if self.outcome == PENDING else self.current + + def summary(self) -> str: + """One line for the apply's output.""" + old = backend_location(self.legacy) + new = backend_location(self.current) + if self.outcome == MIGRATED: + return ( + f"moved {self.resources} resource(s) from {old} to {new} with " + "`tofu init -migrate-state`; the old object is left in place" + ) + if self.outcome == PENDING: + return f"read from {old}: `fluid apply` moves it to {new}" + if self.outcome == OTHER_PROVIDER: + return f"{old} holds {self.detail}, not this provider's; left in place" + return "" + + +def plugin_sources(provider: str) -> FrozenSet[str]: + """The provider sources (``namespace/type``, lower case) ``provider`` emits.""" + from .registry import get_iac_plugin + + plugin = get_iac_plugin(provider) + required = getattr(plugin, "required_providers", None) or {} + sources = {str(spec.get("source", "")).lower() for spec in required.values()} + sources.update(_SOURCE_ALIASES.get(provider, ())) + return frozenset(s for s in sources if s) + + +def state_sources(resources: Iterable[Mapping[str, Any]]) -> FrozenSet[str]: + """``namespace/type`` of every provider the state's resources name.""" + found = set() + for resource in resources: + match = _PROVIDER_ADDR_RE.search(str(resource.get("provider") or "")) + if not match: + # A resource with no readable provider address is itself a reason + # not to decide: keep it visible to the classification. + found.add("") + continue + parts = match.group(1).lower().split("/") + found.add("/".join(parts[-2:])) + return frozenset(found) + + +def classify(resources: Iterable[Mapping[str, Any]], provider: str) -> Tuple[str, str]: + """``(verdict, detail)``: ``"mine"``, ``"other"`` or ``"ambiguous"``. + + ``detail`` names the owner for ``"other"`` and the reason for + ``"ambiguous"``. See the module docstring for the rule. + """ + from .registry import IAC_PLUGINS + + sources = state_sources(resources) + by_plugin = {name: plugin_sources(name) for name in IAC_PLUGINS} + by_plugin.setdefault(provider, plugin_sources(provider)) + known = frozenset().union(*by_plugin.values()) + unknown = sources - known + if unknown: + return "ambiguous", "providers no forge-cli IaC plugin emits: " + ", ".join(sorted(unknown)) + owners = set() + for name, mine in by_plugin.items(): + exclusive = mine - frozenset().union(*(s for n, s in by_plugin.items() if n != name)) + if sources & exclusive: + owners.add(name) + if owners == {provider} and sources <= by_plugin[provider]: + return "mine", provider + if len(owners) == 1 and provider not in owners: + (owner,) = owners + if sources <= by_plugin[owner]: + return "other", f"the {owner} provider's state ({', '.join(sorted(sources))})" + if len(owners) > 1: + return "ambiguous", "resources of several clouds (" + ", ".join(sorted(owners)) + ")" + return "ambiguous", "only providers no single cloud owns (" + ", ".join(sorted(sources)) + ")" + + +def reconcile_state_key( + *, + workdir: Path, + current: Mapping[str, Any], + legacy: Mapping[str, Any], + provider: str, + env: Mapping[str, str], + migrate: bool, + logger: Optional[logging.Logger] = None, +) -> StateReconciliation: + """Make sure ``current`` holds this provider's state, moving it from ``legacy``. + + ``workdir`` is the apply's own workdir, already ``tofu init``-ed on + ``current``: the new key is read there, so an apply whose state is + already at the new key pays one ``tofu state pull`` and nothing else. + Only a new key with no state brings the scratch directory (and its + ``tofu init``, which installs the providers the old state names) in. + ``current`` and ``legacy`` are ``terraform.backend`` blocks + (:func:`.backend.parse_backend`). Returns what was found; raises + :class:`StateMigrationError` when the old state cannot be attributed or + the copy fails or does not verify. ``migrate=False`` reports + :data:`PENDING` instead of copying. + """ + log = logger or _LOG + current_dir = Path(workdir) + now = _pull(current_dir, env) + if now.exists: + return StateReconciliation(CURRENT, legacy, current) + with tempfile.TemporaryDirectory(prefix="fluid-state-") as tmp: + legacy_dir = Path(tmp) / "legacy" + old = _probe(legacy_dir, legacy, env) + if not old.exists or not old.resources: + return StateReconciliation(NOTHING, legacy, current) + verdict, detail = classify(old.resources, provider) + if verdict == "other": + return StateReconciliation(OTHER_PROVIDER, legacy, current, len(old.resources), detail) + if verdict != "mine": + raise StateMigrationError( + "state_migration_ambiguous", + f"{backend_location(legacy)} holds {detail}, and {backend_location(current)} " + f"holds no state, so it cannot be told whether it is the {provider} apply's " + "and nothing was moved. Move it yourself (`tofu init -migrate-state` from a " + "directory configured with the old key), or name the key this apply should " + "use explicitly in --state-backend / FLUID_STATE_BACKEND", + ) + if not migrate: + return StateReconciliation(PENDING, legacy, current, len(old.resources)) + + # Looked at again right before the copy: -force-copy would overwrite + # a state another job wrote since the first probe. + again = _pull(current_dir, env) + if again.exists: + if again.resources == old.resources: + return StateReconciliation(CURRENT, legacy, current) + raise StateMigrationError( + "state_migration_raced", + f"{backend_location(current)} received a different state while this apply " + f"was about to move {backend_location(legacy)} there; nothing was moved", + ) + _write_backend(legacy_dir, current) + init = runner.tofu_init(str(legacy_dir), env=env, force_copy=True) + if not init.ok: + raise StateMigrationError( + "state_migration_failed", + "`tofu init -force-copy` could not copy the state: " + + _tail(init.stderr or init.stdout), + ) + # Verified on content, not lineage: OpenTofu 1.12 writes the copy into + # an empty destination under a fresh lineage and serial 1 (measured + # against an S3 backend), so only the resources can be compared. + moved = _pull(current_dir, env) + if moved.resources != old.resources: + raise StateMigrationError( + "state_migration_unverified", + f"after the copy {backend_location(current)} holds " + f"{len(moved.resources)} resource(s) that are not the " + f"{len(old.resources)} copied from {backend_location(legacy)}; the old " + "object is untouched", + ) + result = StateReconciliation(MIGRATED, legacy, current, len(old.resources)) + log.warning("state_migrated: %s", result.summary()) + return result + + +def _write_backend(workdir: Path, backend: Mapping[str, Any]) -> None: + workdir.mkdir(parents=True, exist_ok=True) + doc = {"terraform": {"backend": dict(backend)}} + (workdir / "main.tf.json").write_text(json.dumps(doc, indent=2), encoding="utf-8") + + +def _probe(workdir: Path, backend: Mapping[str, Any], env: Mapping[str, str]) -> StateDoc: + """Initialise a provider-less module on ``backend`` and pull its state.""" + _write_backend(workdir, backend) + init = runner.tofu_init(str(workdir), env=env) + if not init.ok: + raise StateMigrationError( + "state_migration_probe_failed", + f"could not read {backend_location(backend)}: " + _tail(init.stderr or init.stdout), + ) + return _pull(workdir, env) + + +def _pull(workdir: Path, env: Mapping[str, str]) -> StateDoc: + result = runner.tofu_state_pull(str(workdir), env=env) + if not result.ok: + # stderr only: stdout of `state pull` is the state document, whose + # attributes can hold secrets, and must never reach an error message. + raise StateMigrationError( + "state_migration_probe_failed", + "`tofu state pull` failed: " + (_tail(result.stderr) or f"exit {result.returncode}"), + ) + return parse_state(result.stdout) + + +def parse_state(text: str) -> StateDoc: + """A ``tofu state pull`` document. Empty output is no state.""" + if not (text or "").strip(): + return StateDoc("", 0, ()) + try: + doc = json.loads(text) + except json.JSONDecodeError as exc: + raise StateMigrationError( + "state_migration_probe_failed", f"`tofu state pull` printed no JSON ({exc.msg})" + ) from None + if not isinstance(doc, dict): + raise StateMigrationError( + "state_migration_probe_failed", "`tofu state pull` printed JSON that is not a state" + ) + resources = doc.get("resources") or [] + return StateDoc( + lineage=str(doc.get("lineage") or ""), + serial=int(doc.get("serial") or 0), + resources=tuple(r for r in resources if isinstance(r, dict)), + ) + + +def recorded_backend(workdir: Path) -> Optional[Dict[str, Any]]: + """The backend ``tofu init`` last recorded in ``workdir``, as ``{type: config}``. + + ``.terraform/terraform.tfstate`` keeps it as ``{"backend": {"type": ..., + "config": {...}}}``. ``None`` when there is none or it cannot be read. + """ + path = workdir / ".terraform" / "terraform.tfstate" + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + backend = doc.get("backend") if isinstance(doc, dict) else None + if not isinstance(backend, dict) or not backend.get("type"): + return None + config = backend.get("config") if isinstance(backend.get("config"), dict) else {} + return {str(backend["type"]): config} + + +def records_backend(workdir: Path, backend: Mapping[str, Any]) -> bool: + """True when ``workdir``'s recorded backend is ``backend`` (bucket and key/prefix). + + Only the fields forge-cli writes are compared: OpenTofu records every + backend attribute, most of them null. + """ + recorded = recorded_backend(workdir) + if not recorded or set(recorded) != set(backend): + return False + (kind,) = tuple(backend) + want = backend[kind] or {} + have = recorded[kind] or {} + return all(have.get(k) == v for k, v in want.items()) + + +def _tail(text: str, limit: int = 600) -> str: + text = (text or "").strip() + return text[-limit:] if len(text) > limit else text diff --git a/fluid_build/loader.py b/fluid_build/loader.py index 9e144572..7e07d0f9 100644 --- a/fluid_build/loader.py +++ b/fluid_build/loader.py @@ -19,7 +19,7 @@ import logging import threading from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union try: import yaml # type: ignore @@ -481,18 +481,133 @@ def available_overlay_envs(contract_path: str | Path) -> List[str]: _NOTED_MISSING_OVERLAYS_LOCK = threading.Lock() +#: The ``fluid.workspace.yaml`` key that names, per product, the environments +#: it is deployed to (``{product: [env, ...]}``, the product being the +#: contract's directory name or its id). Written by workspaces whose own gate +#: checks one overlay per environment (fluid-demo-env's ``make targets``); +#: read here so ``--env`` for a declared environment cannot fall back to the +#: base contract. +EXPECTED_ENVIRONMENTS_KEY = "expected-environments" + + +def _base_platforms(contract: Mapping[str, Any]) -> List[str]: + out: List[str] = [] + for expose in contract.get("exposes") or []: + binding = expose.get("binding") if isinstance(expose, dict) else None + platform = binding.get("platform") if isinstance(binding, dict) else None + if platform and str(platform) not in out: + out.append(str(platform)) + return out + + +def declared_environments( + contract_path: str | Path, contract: Mapping[str, Any] +) -> List[Tuple[str, List[str]]]: + """``[(source, envs)]``: where this product's environments are declared. + + Two declarations are read: the contract's own ``environments`` block + (schema ``$defs.environmentConfig``, one key per environment), and the + workspace's ``expected-environments`` entry for this product, looked up + by the contract's directory name, then by its id. Unreadable or absent + declarations are simply not listed. + """ + found: List[Tuple[str, List[str]]] = [] + environments = contract.get("environments") + if isinstance(environments, dict) and environments: + found.append(("the contract's environments block", [str(k) for k in environments])) + try: + from .util.workspace_root import WORKSPACE_CONFIG_FILENAME, find_workspace_root + + base = Path(contract_path).resolve() + root = find_workspace_root(base.parent) + if root is None: + return found + workspace = _parse_file(root / WORKSPACE_CONFIG_FILENAME) + except Exception: # noqa: BLE001 - an unreadable workspace declares nothing + return found + if not isinstance(workspace, dict): + return found + block = workspace.get(EXPECTED_ENVIRONMENTS_KEY) + if not isinstance(block, dict): + return found + for key in (base.parent.name, contract.get("id")): + envs = block.get(key) if isinstance(key, str) else None + if isinstance(envs, list): + found.append( + ( + f"{WORKSPACE_CONFIG_FILENAME} {EXPECTED_ENVIRONMENTS_KEY} ({key})", + [str(e) for e in envs], + ) + ) + break + return found + + +def refuse_declared_missing_overlay( + contract_path: str | Path, env: str, contract: Mapping[str, Any] +) -> None: + """Refuse ``--env `` with no overlay when the product declares ``env``. + + Without an overlay the base contract is used unchanged, which for a + declared environment means deploying it as if it were that environment + (measured: silver ``--env gcp`` validated and planned the local base, + rc=0). The base-by-convention ``dev`` and an env the base contract is + already bound to (``local`` for a local base) are the base, and pass. + """ + if env == BASE_ENV_BY_CONVENTION: + return + platforms = _base_platforms(contract) + if env in platforms: + return + for source, envs in declared_environments(contract_path, contract): + if env in envs: + from ._contract_loader import CLIError + + refusal = CLIError( + 1, + "overlay_declared_but_missing", + { + "env": env, + "contract": str(Path(contract_path)), + "declared_by": source, + "base_platforms": platforms, + "available_envs": available_overlay_envs(contract_path), + "error": ( + f"--env {env!r} has no overlay, but {source} declares {env!r} an " + f"environment of this product, so the base contract (bound to " + f"{', '.join(platforms) or 'nothing'}) would be used as if it were " + f"{env!r}. Add overlays/{env}.yaml, or remove {env!r} from {source}" + ), + }, + ) + # ``str()`` of the error is its sentence, not just the event: most + # callers wrap a load failure as ``{"error": str(e)}``. + refusal.args = (refusal.context["error"],) + raise refusal + + def note_missing_overlay( - contract_path: str | Path, env: str, logger: Optional[logging.Logger] = None + contract_path: str | Path, + env: str, + logger: Optional[logging.Logger] = None, + *, + contract: Optional[Mapping[str, Any]] = None, ) -> None: """Report that ``env`` matched no overlay for ``contract_path``, once. - WARNING for any env but :data:`BASE_ENV_BY_CONVENTION`, naming the env and - the overlays that do exist, because the caller is about to use the base - contract where it asked for an environment. INFO for ``dev``, which is - the base by convention. Emitted once per (contract, env) per process — - one command loads the same contract several times. + WARNING for any env but :data:`BASE_ENV_BY_CONVENTION`, naming the env, + the overlays that do exist and the platforms the base binds to, because + the caller is about to use the base contract where it asked for an + environment. INFO for ``dev``, which is the base by convention. Emitted + once per (contract, env) per process — one command loads the same + contract several times. + + ``contract`` (the base) enables the refusal: an env the product declares + (:func:`refuse_declared_missing_overlay`) is an error, every time. """ log = logger or LOG + if contract is not None: + refuse_declared_missing_overlay(contract_path, env, contract) contract_key = str(Path(contract_path).resolve()) with _NOTED_MISSING_OVERLAYS_LOCK: if (contract_key, env) in _NOTED_MISSING_OVERLAYS: @@ -509,14 +624,21 @@ def note_missing_overlay( ) return existing = available_overlay_envs(contract_key) + platforms = _base_platforms(contract) if contract is not None else [] log.warning( "overlay_not_found: --env %r matched no overlay for %s, so the BASE contract is " - "used unchanged. Overlays that exist: %s. Add an overlay for it under overlays/ " + "used unchanged%s. Overlays that exist: %s. Add an overlay for it under overlays/ " "or pass one of the existing environments.", env, contract_key, + f" (it binds to {', '.join(platforms)}, not to {env!r})" if platforms else "", ", ".join(existing) if existing else "none", - extra={"event": "overlay_not_found", "env": env, "available_envs": existing}, + extra={ + "event": "overlay_not_found", + "env": env, + "available_envs": existing, + "base_platforms": platforms, + }, ) @@ -603,7 +725,7 @@ def load_with_overlay( # No overlay found. This used to be a DEBUG line, so ``--env prod`` # with a typo'd or missing overlay silently deployed the BASE # contract at the default log level. Say so, once per contract/env. - note_missing_overlay(base_path, env, log) + note_missing_overlay(base_path, env, log, contract=base) return base # No env → return base as-is diff --git a/fluid_build/observability/config.py b/fluid_build/observability/config.py index ff004214..e57a1605 100644 --- a/fluid_build/observability/config.py +++ b/fluid_build/observability/config.py @@ -17,9 +17,9 @@ """ import os -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Dict, Optional import yaml @@ -53,6 +53,11 @@ class CommandCenterConfig: retry_attempts: int = 3 batch_size: int = 100 # logs/metrics per batch flush_interval: int = 5 # seconds + #: Extra request headers: the ``Authorization`` a bearer credential needs + #: and the ``X-Organization-Id`` the Command Center scopes a run to. Set + #: by callers that authenticate the way ``fluid publish`` does + #: (``cli/_apply_cc_report.py``); never read from a file here. + headers: Dict[str, str] = field(default_factory=dict) @classmethod def from_environment(cls) -> "CommandCenterConfig": @@ -128,7 +133,8 @@ def is_configured(self) -> bool: Returns: True if URL and API key are set, False otherwise """ - return bool(self.enabled and self.url and self.api_key) + credential = self.api_key or self.headers.get("Authorization") + return bool(self.enabled and self.url and credential) def __repr__(self) -> str: # Mask API key for security @@ -136,7 +142,9 @@ def __repr__(self) -> str: if self.api_key: visible_prefix = self.api_key[:4] masked_key = f"{visible_prefix}***REDACTED***" + # Header names only: an Authorization value is a credential. return ( f"CommandCenterConfig(url={self.url}, api_key={masked_key}, " - f"enabled={self.enabled}, timeout={self.timeout})" + f"enabled={self.enabled}, timeout={self.timeout}, " + f"headers={sorted(self.headers)})" ) diff --git a/fluid_build/observability/reporter.py b/fluid_build/observability/reporter.py index 0b3eb7d8..203cb55e 100644 --- a/fluid_build/observability/reporter.py +++ b/fluid_build/observability/reporter.py @@ -225,13 +225,19 @@ def __init__(self, config: CommandCenterConfig): # Circuit breaker self.circuit_breaker = CircuitBreaker(failure_threshold=5, timeout=60, success_threshold=1) + #: What the worker managed to deliver, for a caller that has to say + #: whether the run reached the Command Center (the queue is async). + self.stats: Dict[str, int] = {"sent": 0, "failed": 0} + # Session for connection pooling self.session: Optional[Any] = None if self.enabled and requests: self.session = requests.Session() - self.session.headers.update( - {"X-API-Key": config.api_key, "Content-Type": "application/json"} - ) + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["X-API-Key"] = config.api_key + headers.update(config.headers or {}) + self.session.headers.update(headers) def start(self): """Start background worker thread.""" @@ -477,7 +483,17 @@ def _send_event(self, event: Dict[str, Any]): response.raise_for_status() logger.debug(f"Command Center: {method} {endpoint} → {response.status_code}") + self.stats["sent"] += 1 except Exception as e: - logger.warning(f"Failed to send event to Command Center: {e}") + self.stats["failed"] += 1 + # The type and the status only: a requests exception's text + # repeats the URL, and nothing here needs the response body. + status = getattr(getattr(e, "response", None), "status_code", None) + logger.warning( + "Failed to send event to Command Center: %s %s -> %s", + method, + endpoint, + status or type(e).__name__, + ) raise diff --git a/fluid_build/policy/sovereignty.py b/fluid_build/policy/sovereignty.py index 6caa7f5d..3efbb8c7 100644 --- a/fluid_build/policy/sovereignty.py +++ b/fluid_build/policy/sovereignty.py @@ -27,7 +27,7 @@ from enum import Enum from pathlib import Path from types import MappingProxyType -from typing import Any, Dict, List, Mapping, Optional, Tuple +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple from ._common import iter_exposes @@ -182,6 +182,7 @@ def region_jurisdiction_map() -> Mapping[str, str]: for provider in ("aws", "gcp", "azure"): table.update(_load_vendored(provider)) table.update(SovereigntyValidator.VENDORED_CORRECTIONS) + table.update(_MULTI_REGION_JURISDICTIONS) table.update(_load_botocore_aws()) # Read-only: the cached object is shared process-wide (and re-exported by # providers/aws/util/sovereignty.py), so a stray mutation anywhere would @@ -370,12 +371,25 @@ def validate(self, contract: Dict[str, Any]) -> Tuple[bool, List[SovereigntyViol Returns: (is_valid, violations) - is_valid=False means BLOCK deployment in strict mode """ - violations = [] - # Extract sovereignty config (optional in 0.7.1) sovereignty = contract.get("sovereignty") if not sovereignty: return True, [] # No sovereignty constraints = always valid + return self.check_placements(sovereignty, contract_placements(contract)) + + def check_placements( + self, sovereignty: Mapping[str, Any], placements: Sequence[Tuple[str, Optional[str]]] + ) -> Tuple[bool, List[SovereigntyViolation]]: + """Evaluate ``sovereignty`` against each ``(where, region)`` placement. + + :meth:`validate` passes the contract's own bindings + (:func:`contract_placements`); a provider hook passes the places its + emitted resources actually land (the GCP plugin passes every resource + ``location``), so a region the provider filled in by default is + checked where it is used. A placement whose region is ``None`` is a + binding on a region-placed platform that names none. + """ + violations = [] # Defaults MUST mirror the JSON schema's declared ``default`` keys # (``$defs.sovereignty`` in fluid-schema-0.7.x.json). They previously @@ -397,16 +411,36 @@ def validate(self, contract: Dict[str, Any]) -> Tuple[bool, List[SovereigntyViol "crossBorderTransfer", DEFAULT_CROSS_BORDER_TRANSFER ) - # Validate each expose's binding location - for expose in iter_exposes(contract): - binding = expose.get("binding", {}) - location = binding.get("location", {}) - region = location.get("region") - + # Validate each place the contract puts data + for expose_id, region in placements: + # Check 0: a region-placed binding that names no region. It used + # to be skipped ("no region, nothing to check"), which failed OPEN: + # validate and ``plan --check-sovereignty`` printed PASS while the + # platform picked the region itself (BigQuery: the US multi-region, + # measured on an EU-only contract). The mode decides, like check 2: + # strict refuses, advisory warns, audit logs. if not region: - continue # No region specified, skip validation - - expose_id = expose.get("exposeId", "unknown") + violations.append( + SovereigntyViolation( + severity=severity_for(enforcement_mode), + message=( + "Binding declares no region, so where its data lives cannot " + "be checked against the sovereignty policy (the platform " + "would choose)" + ), + expose_id=expose_id, + region_expected=allowed_regions or None, + suggestion=( + "Set binding.location.region" + + ( + f" to one of: {', '.join(allowed_regions)}" + if allowed_regions + else "" + ) + ), + ) + ) + continue # Check 1: Denied regions — deliberately an error in EVERY mode. # @@ -489,11 +523,9 @@ def validate(self, contract: Dict[str, Any]) -> Tuple[bool, List[SovereigntyViol # than being silently folded into a jurisdiction comparison. if data_residency and not cross_border_transfer: baseline: Any = _UNSET - for exp in iter_exposes(contract): - exp_region = exp.get("binding", {}).get("location", {}).get("region") + for exp_id, exp_region in placements: if not exp_region: continue - exp_id = exp.get("exposeId", "unknown") exp_jurisdiction = region_jurisdiction_map().get(exp_region, "Unknown") if exp_jurisdiction == "Unknown": @@ -552,6 +584,56 @@ def validate(self, contract: Dict[str, Any]) -> Tuple[bool, List[SovereigntyViol return is_valid, violations +#: Platforms whose bindings put data in a cloud region: a binding on one of +#: these with a sovereignty block and no region is a finding (check 0), not a +#: skip. Other platforms (``local`` above all, and those whose region lives +#: outside the binding) keep the old behaviour: no region, nothing checked. +REGION_PLACED_PLATFORMS = frozenset({"aws", "gcp", "azure"}) + +#: Multi-region locations the vendored region table does not carry. BigQuery +#: and Cloud Storage both name their multi-regions ``US`` (data centres in the +#: United States) and ``EU`` (data centres in EU member states); left unmapped +#: they resolved "Unknown", so the ``US`` a GCP binding with no region used to +#: land in could not fail a ``jurisdiction: EU`` check. +_MULTI_REGION_JURISDICTIONS = {"US": "US", "EU": "EU", "us": "US", "eu": "EU"} + + +def binding_region(binding: Mapping[str, Any]) -> Optional[str]: + """The region a binding places its data in, read where its emitter reads it. + + ``location.region``, and for GCP also ``location.location``: the GCP + emitter falls back to it (``iac/providers/gcp.py``), so a check that read + ``region`` alone never saw a BigQuery dataset placed through ``location``. + """ + location = binding.get("location") if isinstance(binding, Mapping) else None + if not isinstance(location, Mapping): + return None + region = location.get("region") + if not region and str(binding.get("platform") or "").lower() == "gcp": + region = location.get("location") + return str(region) if region else None + + +def contract_placements(contract: Mapping[str, Any]) -> List[Tuple[str, Optional[str]]]: + """``(exposeId, region)`` for each expose the sovereignty checks evaluate. + + An expose with a region is always listed. One without is listed with + ``None`` only on a :data:`REGION_PLACED_PLATFORMS` platform, where the + region is the platform's to pick; any other binding with no region is + left out, as before. + """ + out: List[Tuple[str, Optional[str]]] = [] + for expose in iter_exposes(dict(contract)): + binding = expose.get("binding") or {} + if not isinstance(binding, Mapping): + continue + region = binding_region(binding) + platform = str(binding.get("platform") or "").lower() + if region or platform in REGION_PLACED_PLATFORMS: + out.append((str(expose.get("exposeId", "unknown")), region)) + return out + + def validate_sovereignty(contract: Dict[str, Any]) -> Tuple[bool, List[str]]: """ Convenience function for CLI integration. diff --git a/fluid_build/providers/gcp/provider.py b/fluid_build/providers/gcp/provider.py index a7c24ce1..8bdf6196 100644 --- a/fluid_build/providers/gcp/provider.py +++ b/fluid_build/providers/gcp/provider.py @@ -186,6 +186,12 @@ def plan( # Older planner signature without mode kwarg. actions = plan_actions(contract, self.project, self.region, self.logger) + # Sovereignty, the way AwsProvider.plan does it: a refusal raised + # here reaches `fluid apply` / `fluid generate iac` through + # native_actions, which re-raises a sovereignty veto instead of + # treating it as "planner unavailable". + self._validate_sovereignty(contract, actions) + self.info_kv( event="plan_completed", contract_id=contract.get("id"), @@ -195,10 +201,36 @@ def plan( return actions + except ProviderError: + raise except Exception as e: self.err_kv(event="plan_failed", contract_id=contract.get("id"), error=str(e)) raise ProviderError(f"Failed to plan GCP deployment: {e}") from e + def _validate_sovereignty( + self, contract: Mapping[str, Any], actions: List[Dict[str, Any]] + ) -> None: + """Refuse a planned placement outside ``contract.sovereignty``. + + Checks each gcp expose's binding region (none is a finding under + strict) and every ``location`` / ``region`` a planned action carries: + where the planner fell back to a default (``US`` for a dataset, this + provider's region for a scheduler job or a staging bucket), that + default is what gets checked. See ``util/sovereignty.py``. + """ + if not contract.get("sovereignty"): + return + from fluid_build._errors import ResidencyViolationError, SovereigntyViolationError + + from .util.sovereignty import action_placements, enforce_gcp_sovereignty + + try: + enforce_gcp_sovereignty(contract, action_placements(actions), logger=self.logger) + except (SovereigntyViolationError, ResidencyViolationError) as e: + self.err_kv(event="sovereignty_violation", error=str(e)) + raise ProviderError(str(e)) from e + self.info_kv(event="sovereignty_validated", region=self.region) + def apply(self, actions: List[Dict[str, Any]], **kwargs: Any) -> ApplyResult: """Native GCP apply is retired — GCP uses the OpenTofu engine. diff --git a/fluid_build/providers/gcp/util/sovereignty.py b/fluid_build/providers/gcp/util/sovereignty.py new file mode 100644 index 00000000..018bfebf --- /dev/null +++ b/fluid_build/providers/gcp/util/sovereignty.py @@ -0,0 +1,166 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The GCP provider's sovereignty hook: refuse a placement outside the policy. + +AWS refuses an out-of-jurisdiction region inside its planner +(``AwsProvider._validate_sovereignty``), and ``fluid generate iac`` / +``fluid apply`` recognise that refusal through ``native_actions``. GCP had no +such hook, so only ``fluid validate`` stood between a contract and a dataset +in the wrong jurisdiction, and a binding with no region went through every +stage and landed in BigQuery's ``US`` multi-region. + +The check is the policy engine's own (``policy.sovereignty``, +``check_placements``), not a second rule set, applied to where the data +actually goes rather than to what the binding says: + +* each gcp expose whose binding names no region (the engine's check 0: the + mode decides, strict refuses); +* each ``location`` / ``region`` a resource carries: every resource the + OpenTofu plugin emits (``GcpIacPlugin.emit``, the chokepoint of ``fluid + apply`` and ``fluid generate iac``, which runs whether or not a native + provider can be built), and every action the native planner produced + (``GcpProvider.plan``). That is where a region the provider filled in by + default (the planner's ``US`` dataset default, the provider region a + Cloud Scheduler job or a staging bucket inherits, ``--region``'s + ``europe-west3`` or the SDK's ``us-central1``) meets ``allowedRegions``. + +Error-severity findings raise :class:`~fluid_build._errors.SovereigntyViolationError` +(the typed error ``generate_iac._is_sovereignty_refusal`` recognises through +a ``ProviderError`` cause chain); warnings and info are logged and pass, +which is what ``advisory`` and ``audit`` mean. +""" + +from __future__ import annotations + +import logging +from typing import Any, Iterable, List, Mapping, Optional, Sequence, Tuple + +from fluid_build.policy.sovereignty import ( + SovereigntyValidator, + SovereigntyViolation, + binding_region, +) + +_LOG = logging.getLogger(__name__) + +Placement = Tuple[str, Optional[str]] + + +def unplaced_gcp_exposes(contract: Mapping[str, Any]) -> List[Placement]: + """``(exposeId, None)`` for each gcp expose whose binding names no region.""" + out: List[Placement] = [] + for expose in contract.get("exposes") or []: + if not isinstance(expose, Mapping): + continue + binding = expose.get("binding") or {} + if not isinstance(binding, Mapping): + continue + if str(binding.get("platform") or "").lower() != "gcp": + continue + if binding_region(binding) is None: + out.append((str(expose.get("exposeId", "unknown")), None)) + return out + + +def resource_placements(resources: Mapping[str, Any]) -> List[Placement]: + """``(address, location)`` for every emitted resource that names one. + + ``resources`` is the plugin's ``{type: {name: body}}``. A value that is an + OpenTofu reference (``${...}``) is not a place and is skipped. + """ + out: List[Placement] = [] + for rtype, by_name in (resources or {}).items(): + if not isinstance(by_name, Mapping): + continue + for name, body in by_name.items(): + if not isinstance(body, Mapping): + continue + for key in ("location", "region"): + value = body.get(key) + if isinstance(value, str) and value and not value.startswith("${"): + out.append((f"{rtype}.{name}", value)) + break + return out + + +def action_placements(actions: Iterable[Mapping[str, Any]]) -> List[Placement]: + """``(action id, location)`` for every planned action that names one.""" + out: List[Placement] = [] + for index, action in enumerate(actions or ()): + if not isinstance(action, Mapping): + continue + for key in ("location", "region"): + value = action.get(key) + if isinstance(value, str) and value: + where = str(action.get("id") or action.get("op") or f"action[{index}]") + out.append((where, value)) + break + return out + + +def gcp_sovereignty_violations( + contract: Mapping[str, Any], placements: Sequence[Placement] +) -> List[SovereigntyViolation]: + """The engine's findings for this contract's gcp exposes and ``placements``.""" + sovereignty = contract.get("sovereignty") + if not isinstance(sovereignty, Mapping) or not sovereignty: + return [] + everything = unplaced_gcp_exposes(contract) + list(placements) + if not everything: + return [] + _, violations = SovereigntyValidator().check_placements(sovereignty, everything) + return violations + + +def enforce_gcp_sovereignty( + contract: Mapping[str, Any], + placements: Sequence[Placement], + *, + logger: Optional[logging.Logger] = None, +) -> None: + """Raise on an error-severity finding; log the rest.""" + from fluid_build._errors import SovereigntyViolationError, doc_url + + log = logger or _LOG + violations = gcp_sovereignty_violations(contract, placements) + errors = [v for v in violations if v.severity == "error"] + for v in violations: + if v.severity != "error": + log.warning("gcp sovereignty (%s): %s: %s", v.severity, v.expose_id, v.message) + if not errors: + return + sovereignty = contract.get("sovereignty") or {} + allowed = [str(r) for r in (sovereignty.get("allowedRegions") or [])] + # ``where: message``, de-duplicated in order. Not ``[where]``: the CLI + # renders errors through rich, which reads square brackets as markup. + findings = "; ".join(dict.fromkeys(f"{v.expose_id}: {v.message}" for v in errors)) + raise SovereigntyViolationError( + what=f"GCP placement refused by the sovereignty policy: {findings}", + why=( + "contract.sovereignty is enforced " + f"({sovereignty.get('enforcementMode', 'strict')}) and " + + ( + f"allows {', '.join(allowed)}" + if allowed + else f"requires jurisdiction {sovereignty.get('jurisdiction')!r}" + ) + + "; this is where the GCP resources would be created." + ), + fix=( + "Set binding.location.region on every gcp expose to an allowed region " + "(the BigQuery dataset and bucket take it), or change the sovereignty policy." + ), + doc=doc_url("sovereignty"), + ) diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index df16ffc5..3ccb4f3d 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -497,8 +497,16 @@ def validate_env_name(raw: Any) -> str: return validate_id(raw, kind="env") -def dag_id_for(product_id: str, build_id: str) -> str: - dag_id = f"{product_id}__{build_id}" +def dag_id_for(product_id: str, build_id: str, env: Optional[str] = None) -> str: + """``__``, or ``____`` for an env's DAG. + + The env is part of the id because one contract deployed to two clouds + (``--env aws`` and ``--env gcp`` overlays) is one product id: both DAGs + had the same id, and in one Airflow the second to parse replaced the + first. Airflow keys run history on the id, so an env-bound DAG from an + earlier release starts a new history under its new id. + """ + dag_id = f"{product_id}__{env}__{build_id}" if env else f"{product_id}__{build_id}" if len(dag_id) > _MAX_DAG_ID: raise ScheduleRenderError( f"dag_id {dag_id!r} exceeds Airflow's {_MAX_DAG_ID}-character limit; " @@ -507,6 +515,18 @@ def dag_id_for(product_id: str, build_id: str) -> str: return dag_id +def schedule_scope_for(product_id: str, env: Optional[str] = None) -> str: + """The directory an env's DAGs live in, under the schedule artifacts root. + + ```` with no env, ``__`` with one. ``fluid + schedule-sync --delete-scope product`` mirrors each such directory onto + the same-named one at the scheduler, deleting what the source lacks, so + the aws and the gcp DAGs of one product need a directory each or each + sync deletes the other's. + """ + return f"{product_id}__{env}" if env else product_id + + def dag_filename_for(build_id: str) -> str: # A plain module name: dots in a file name make Airflow import the DAG # under a dotted module path. @@ -644,7 +664,7 @@ def render_dag( ")\n" "\n" "with DAG(\n" - f" dag_id={lit(dag_id_for(product_id, build.build_id))},\n" + f" dag_id={lit(dag_id_for(product_id, build.build_id, env))},\n" f" description={lit(f'fluid apply {product_id} --build-id {build.build_id}')},\n" " schedule=SCHEDULE,\n" " start_date=pendulum.datetime(2026, 1, 1, tz=TIMEZONE),\n" @@ -727,6 +747,7 @@ def render_fluid_apply_dags( "has_scheduled_builds", "render_dag", "render_fluid_apply_dags", + "schedule_scope_for", "scheduled_builds", "uses_fluid_apply_dags", "validate_contract_path", diff --git a/tests/cli/test_apply_reports_to_command_center.py b/tests/cli/test_apply_reports_to_command_center.py new file mode 100644 index 00000000..36aa96e1 --- /dev/null +++ b/tests/cli/test_apply_reports_to_command_center.py @@ -0,0 +1,376 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid apply`` reports each run to the Command Center, against a stub server. + +A run is registered at ``POST /api/v1/executions`` and closed at ``PATCH +/api/v1/executions/{id}``, the Command Center's executions API +(``app/api/v1/executions.py``: ``ExecutionCreate`` / ``ExecutionUpdate``), +with the credential and the organization ``fluid publish`` uses. Before this, +``get_reporter`` had no callers and a Jenkins apply left no run behind. + +The stub is a real HTTP server on loopback that records every request. The +apply goes through the real parser and the real OpenTofu engine; only the +``tofu`` binary is stubbed (init, plan and apply answer with the change +summary a real run prints), and the native planner is skipped. Pinned: what +a run says (product, contract version, environment, provider, resources, +counts, timings, status), that the organization comes from the publish +configuration (id or slug), that the credential is only ever a header, that +tofu's own output never reaches the Command Center, and that an outage, a +refusal or a 500 never changes the apply's exit code. +""" + +from __future__ import annotations + +import json +import logging +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, Iterator, List + +import pytest +import yaml + +from fluid_build.cli import _apply_opentofu_engine as engine +from fluid_build.cli._common import CLIError +from fluid_build.iac import runner + +pytestmark = pytest.mark.unit + +_LOG = logging.getLogger("test.apply_cc_report") +_KEY = "cc-test-key-not-a-secret" # pragma: allowlist secret +_TOFU_OUTPUT_MARKER = "tofu-printed-this-attribute-value" +_ORG = "0f5e3c1a-telco" + + +class _Recorder: + def __init__(self) -> None: + self.requests: List[Dict[str, Any]] = [] + self.status = {"POST": 201, "PATCH": 200, "GET": 200} + + +@pytest.fixture +def cc() -> Iterator[Any]: + """A stub Command Center on loopback: ``(base_url, recorder)``.""" + recorder = _Recorder() + + class Handler(BaseHTTPRequestHandler): + def _answer(self, method: str) -> None: + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"" + recorder.requests.append( + { + "method": method, + "path": self.path, + "headers": {k: v for k, v in self.headers.items()}, + "raw": raw.decode("utf-8"), + "body": json.loads(raw) if raw else None, + } + ) + status = recorder.status[method] + if method == "GET" and self.path == "/api/v1/organizations": + payload: Any = [{"id": _ORG, "slug": "northwind-telco", "name": "Telco"}] + else: + payload = {"ok": status < 400} + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self): # noqa: N802 + self._answer("POST") + + def do_PATCH(self): # noqa: N802 + self._answer("PATCH") + + def do_GET(self): # noqa: N802 + self._answer("GET") + + def log_message(self, *_a): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}", recorder + finally: + server.shutdown() + server.server_close() + + +_CONTRACT = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "bronze.customer_subscriptions", + "name": "Customer Subscriptions", + "version": "1.4.0", + "description": "Run report fixture.", + "domain": "Customer", + "metadata": {"layer": "Bronze", "owner": {"team": "data-platform", "email": "dp@example.com"}}, + "exposes": [ + { + "exposeId": "subscriptions", + "kind": "table", + "binding": { + "platform": "local", + "format": "parquet", + "location": {"path": "data/customer_subscriptions.parquet"}, + }, + "contract": {"schema": [{"name": "subscription_id", "type": "STRING"}]}, + } + ], +} + +_GCP_OVERLAY = { + "exposes": [ + { + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "northwind-demo", + "dataset": "demo_bronze", + "table": "customer_subscriptions", + "region": "europe-west1", + }, + } + } + ] +} + + +@pytest.fixture +def product(tmp_path: Path, monkeypatch) -> Path: + for var in ( + "FLUID_CC_ENDPOINT", + "FLUID_CATALOG_FLUID_CC_URL", + "FLUID_API_KEY", + "FLUID_BEARER_TOKEN", + "FLUID_CC_ORG_ID", + "FLUID_COMMAND_CENTER_URL", + "FLUID_COMMAND_CENTER_API_KEY", + "FLUID_COMMAND_CENTER_ENABLED", + "FLUID_STATE_BACKEND", + "FLUID_PROVIDER", + "JENKINS_URL", + "BUILD_TAG", + "GOOGLE_APPLICATION_CREDENTIALS", + ): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + (tmp_path / "overlays").mkdir() + (tmp_path / "overlays" / "gcp.yaml").write_text(yaml.safe_dump(_GCP_OVERLAY), encoding="utf-8") + contract = tmp_path / "contract.fluid.yaml" + contract.write_text(yaml.safe_dump(_CONTRACT, sort_keys=False), encoding="utf-8") + return contract + + +def _summary(add: int) -> List[Dict[str, Any]]: + return [{"type": "change_summary", "changes": {"add": add, "change": 0, "remove": 0}}] + + +@pytest.fixture +def tofu(monkeypatch) -> Dict[str, Any]: + """The ``tofu`` binary, answered: init ok, plan +2, apply +2.""" + behaviour: Dict[str, Any] = {"apply_ok": True} + ok = runner.TofuResult + monkeypatch.setattr(runner, "tofu_path", lambda: "/usr/bin/tofu") + monkeypatch.setattr(runner, "require_tofu_version", lambda *a, **k: None) + monkeypatch.setattr(runner, "tofu_init", lambda *a, **k: ok("init", 0, "", "")) + monkeypatch.setattr(runner, "tofu_state_list", lambda *a, **k: []) + monkeypatch.setattr(runner, "tofu_state_resources", lambda *a, **k: []) + monkeypatch.setattr(runner, "tofu_prior_state_resources", lambda *a, **k: []) + monkeypatch.setattr( + runner, "tofu_import", lambda *a, **k: ok("import", 1, "", "not found (stub)") + ) + monkeypatch.setattr( + runner, "tofu_plan", lambda *a, **k: ok("plan", 0, "", "", events=_summary(2)) + ) + + def _apply(*_a, **_k): + if behaviour["apply_ok"]: + return ok("apply", 0, "", "", events=_summary(2)) + return ok("apply", 1, "", f"Error: googleapi: 403 {_TOFU_OUTPUT_MARKER}") + + monkeypatch.setattr(runner, "tofu_apply", _apply) + monkeypatch.setattr(engine, "native_actions", lambda contract, logger: []) + return behaviour + + +def _apply(contract: Path, *extra: str) -> int: + from fluid_build.cli import build_parser + + args = build_parser().parse_args( + ["apply", str(contract), "--env", "gcp", "--yes", "--no-verify-federation", *extra] + ) + return args.func(args, _LOG) + + +def _by(recorder: _Recorder, method: str) -> List[Dict[str, Any]]: + return [r for r in recorder.requests if r["method"] == method] + + +def _configure(monkeypatch, url: str, *, org: bool = True) -> None: + monkeypatch.setenv("FLUID_CC_ENDPOINT", url) + monkeypatch.setenv("FLUID_API_KEY", _KEY) + if org: + monkeypatch.setenv("FLUID_CC_ORG_ID", _ORG) + + +def test_an_apply_is_registered_and_closed_with_what_it_did(product, tofu, cc, monkeypatch, capsys): + url, recorder = cc + _configure(monkeypatch, url) + + assert _apply(product) == 0 + + (post,) = _by(recorder, "POST") + (patch,) = _by(recorder, "PATCH") + assert post["path"] == "/api/v1/executions" + body = post["body"] + assert body["command"] == "apply" + assert body["status"] == "running" + assert body["provider"] == "gcp" + assert body["environment"] == "gcp" + assert body["runner"] == "cli" + meta = body["metadata"] + assert meta["product_id"] == "bronze.customer_subscriptions" + assert meta["contract_version"] == "1.4.0" + assert meta["fluid_version"] == "0.7.5" + assert meta["platform"] == "gcp" + assert meta["environment"] == "gcp" + assert len(meta["contract_hash"]) == 64 + assert meta["state"].startswith("local: ") + + assert patch["path"] == f"/api/v1/executions/{body['execution_id']}" + update = patch["body"] + assert update["status"] == "success" + result = update["result"] + assert result["planned_changes"] == {"add": 2, "change": 0, "remove": 0} + assert result["applied_changes"] == {"add": 2, "change": 0, "remove": 0} + assert result["dry_run"] is False + assert result["exit_code"] == 0 + assert result["duration_seconds"] >= 0 + assert result["started_at"] <= result["finished_at"] + assert "google_bigquery_dataset.bronze_customer_subscriptions_demo_bronze" in ( + result["resources"] + ) + assert "google_bigquery_table.bronze_customer_subscriptions_customer_subscriptions" in ( + result["resources"] + ) + + # The organization and the credential travel as headers, and only there. + for request in (post, patch): + assert request["headers"]["X-API-Key"] == _KEY + assert request["headers"]["X-Organization-Id"] == _ORG + assert _KEY not in request["raw"] + assert f"command center: run {body['execution_id']} reported" in capsys.readouterr().out + + +def test_the_organization_named_by_slug_in_fluid_config_is_resolved(product, tofu, cc, monkeypatch): + """The demo lab names each product's organization by slug in its own + fluid.config.yaml and never sets FLUID_CC_ORG_ID.""" + url, recorder = cc + _configure(monkeypatch, url, org=False) + (product.parent / "fluid.config.yaml").write_text( + yaml.safe_dump({"catalogs": {"fluid-command-center": {"organization": "northwind-telco"}}}), + encoding="utf-8", + ) + + assert _apply(product) == 0 + + assert [r["path"] for r in _by(recorder, "GET")] == ["/api/v1/organizations"] + for request in _by(recorder, "POST") + _by(recorder, "PATCH"): + assert request["headers"]["X-Organization-Id"] == _ORG + + +def test_a_failed_apply_is_closed_failed_and_tofu_output_stays_home(product, tofu, cc, monkeypatch): + url, recorder = cc + _configure(monkeypatch, url) + tofu["apply_ok"] = False + + with pytest.raises(CLIError) as exc: + _apply(product) + assert exc.value.event == "opentofu_apply_failed" + + (patch,) = _by(recorder, "PATCH") + assert patch["body"]["status"] == "failed" + assert patch["body"]["result"]["error_event"] == "opentofu_apply_failed" + assert patch["body"]["result"]["applied_changes"] is None + for request in recorder.requests: + assert _TOFU_OUTPUT_MARKER not in request["raw"] + + +@pytest.mark.parametrize("status", [500, 401]) +def test_a_command_center_that_refuses_never_changes_the_exit_code( + product, tofu, cc, monkeypatch, capsys, status +): + url, recorder = cc + _configure(monkeypatch, url) + recorder.status.update(POST=status, PATCH=status) + + assert _apply(product) == 0 + assert "not fully reported" in capsys.readouterr().out + + +def test_a_command_center_that_is_down_never_changes_the_exit_code( + product, tofu, monkeypatch, capsys +): + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + closed = probe.getsockname()[1] + _configure(monkeypatch, f"http://127.0.0.1:{closed}") + monkeypatch.setenv("FLUID_COMMAND_CENTER_TIMEOUT", "1") + + assert _apply(product) == 0 + assert "not fully reported" in capsys.readouterr().out + + +def test_nothing_is_sent_or_said_when_no_command_center_is_configured(product, tofu, cc, capsys): + _url, recorder = cc + assert _apply(product) == 0 + assert recorder.requests == [] + assert "command center" not in capsys.readouterr().out + + +def test_the_opt_out_sends_nothing(product, tofu, cc, monkeypatch): + url, recorder = cc + _configure(monkeypatch, url) + monkeypatch.setenv("FLUID_COMMAND_CENTER_ENABLED", "false") + + assert _apply(product) == 0 + assert recorder.requests == [] + + +def test_a_private_address_is_refused_before_the_credential_is_sent( + product, tofu, monkeypatch, capsys +): + from fluid_build.providers.catalogs.fluid_cc import FluidCommandCenterProvider + + called: List[str] = [] + monkeypatch.setattr( + FluidCommandCenterProvider, + "_list_organizations", + lambda self: called.append("organizations"), + ) + _configure(monkeypatch, "http://10.20.30.40:5200", org=False) + + assert _apply(product) == 0 + assert called == [] + assert "private or metadata address" in " ".join(capsys.readouterr().out.split()) diff --git a/tests/cli/test_diff_state_drift.py b/tests/cli/test_diff_state_drift.py index 91374471..137b8a9d 100644 --- a/tests/cli/test_diff_state_drift.py +++ b/tests/cli/test_diff_state_drift.py @@ -151,8 +151,17 @@ def __init__(self, monkeypatch, *, plan_rc: int = 0, doc: Optional[Dict] = None) from fluid_build.cli import _apply_opentofu_engine as engine monkeypatch.setattr(engine, "native_actions", lambda contract, logger: []) + # Moving a pre-provider-key state (``iac.state_migration``) is not + # under test here: nothing to move. + from fluid_build.iac.state_migration import CURRENT, StateReconciliation - def _init(self, workdir, *, backend=True, env=None): + monkeypatch.setattr( + engine, + "_reconcile_state", + lambda **kw: StateReconciliation(CURRENT, kw["legacy"], kw["current"]), + ) + + def _init(self, workdir, *, backend=True, env=None, reconfigure=False, force_copy=False): self.calls.append(f"init backend={backend}") self.module_during_run = (Path(workdir) / "main.tf.json").read_text(encoding="utf-8") return runner.TofuResult("init", 0, "", "") @@ -351,7 +360,7 @@ def test_the_state_backend_is_resolved_as_apply_resolves_it(workspace, monkeypat assert _invoke(["diff", str(contract), "--out", str(out)]) == (0, None) assert tofu.calls[0] == "init backend=True" assert '"s3"' in tofu.module_during_run - assert _state(out)["state"] == f"remote: s3://team-state/fluid/{CID}/terraform.tfstate" + assert _state(out)["state"] == f"remote: s3://team-state/fluid/{CID}/aws/terraform.tfstate" # The module the pass wrote into a fresh workdir is not left there. assert not (_workdir(workspace) / "main.tf.json").exists() diff --git a/tests/cli/test_generate_schedule_fluid_apply.py b/tests/cli/test_generate_schedule_fluid_apply.py index 3d34ece1..a6f5077d 100644 --- a/tests/cli/test_generate_schedule_fluid_apply.py +++ b/tests/cli/test_generate_schedule_fluid_apply.py @@ -419,6 +419,10 @@ def _literal_connection(contract: str = DEMO_CONTRACT) -> str: #: Variables the ``fluid apply`` code path reads that a scheduled run does not #: need, so the worker environment does not pass them. NOT_PASSED = { + "JENKINS_URL": ( + "only tags a Command Center run report as a Jenkins run; a scheduled run is not one" + ), + "ProgramData": "the Windows system config directory; the DAG task runs under bash", "PRODUCTION": "only chooses a --safe-mode tip in the CLI banner", "SOURCE_DATE_EPOCH": "tar mtimes for `fluid bundle`, which a scheduled apply never runs", "USERPROFILE": "the Windows home directory; the DAG task runs under bash", diff --git a/tests/cli/test_one_contract_two_clouds_pipeline.py b/tests/cli/test_one_contract_two_clouds_pipeline.py new file mode 100644 index 00000000..9d5e953e --- /dev/null +++ b/tests/cli/test_one_contract_two_clouds_pipeline.py @@ -0,0 +1,264 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One contract, two clouds, one pipeline generator: the small collisions. + +* Stage 11: the aws and the gcp DAG of one product had the same ``dag_id`` + (``__``) and the same schedule directory, so in one + Airflow the second sync replaced (and ``--delete-scope product`` deleted) + the first. The env is now in both. +* Stage 6: the generated plan stage did not run ``--check-sovereignty``, for + any CI system, so a strict sovereignty violation first failed at apply. +* ``--env gcp`` with no gcp overlay applied the local base unchanged with a + warning (measured: silver validate and plan rc 0). When the product + declares gcp (its ``environments`` block, or the workspace's + ``expected-environments``, which fluid-demo-env keeps) it is now an error. +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +import pytest +import yaml + +from fluid_build.schedulers.airflow import fluid_apply +from tests.cli._schedule_dag_fixtures import DEMO_CONTRACT, DEMO_CONTRACT_PATH, load_dag + +pytestmark = pytest.mark.unit + +_LOG = logging.getLogger("test.one_contract_two_clouds") +_PRODUCT = "bronze.customer_subscriptions" + + +# ── Stage 11: the DAG id and the schedule directory name the env ───────── + + +def _dags(env): + contract = yaml.safe_load(DEMO_CONTRACT) + return fluid_apply.render_fluid_apply_dags(contract, env=env, contract_path=DEMO_CONTRACT_PATH) + + +def test_the_aws_and_the_gcp_dag_of_one_product_have_different_ids(monkeypatch): + ids = {} + for env in ("aws", "gcp"): + (source,) = _dags(env).values() + ids[env] = load_dag(source, monkeypatch).dag["dag_id"] + assert ids == { + "aws": f"{_PRODUCT}__aws__ingest_subscriptions", + "gcp": f"{_PRODUCT}__gcp__ingest_subscriptions", + } + + +def test_a_dag_with_no_env_keeps_its_id(monkeypatch): + (source,) = _dags(None).values() + assert load_dag(source, monkeypatch).dag["dag_id"] == f"{_PRODUCT}__ingest_subscriptions" + + +def test_each_env_gets_its_own_schedule_directory(tmp_path, monkeypatch): + """schedule-sync --delete-scope product mirrors each directory with delete: + one directory per env is what keeps the aws sync off the gcp DAG.""" + from fluid_build.cli import generate_artifacts + from tests.cli._schedule_dag_fixtures import write_project + + monkeypatch.delenv("FLUID_ENV", raising=False) + write_project(tmp_path) + contract_dir = (tmp_path / DEMO_CONTRACT_PATH).parent + (contract_dir / "overlays" / "gcp.yaml").write_text( + yaml.safe_dump( + { + "exposes": [ + { + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "p", + "dataset": "d", + "table": "t", + "region": "europe-west1", + }, + } + } + ] + } + ), + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + for env in ("aws", "gcp"): + parser = argparse.ArgumentParser() + generate_artifacts.register_subcommand(parser.add_subparsers()) + argv = ["artifacts", DEMO_CONTRACT_PATH, "--out", f"dist-{env}", "--env", env] + assert generate_artifacts.run(parser.parse_args(argv), _LOG) == 0 + scopes = sorted(p.name for p in (tmp_path / f"dist-{env}" / "schedule").iterdir()) + assert scopes == [f"{_PRODUCT}__{env}"] + + +# ── Stage 6: every generated plan stage checks sovereignty ──────────────── + + +def _strings(node): + if isinstance(node, str): + yield node + elif isinstance(node, dict): + for value in node.values(): + yield from _strings(value) + elif isinstance(node, list): + for value in node: + yield from _strings(value) + + +def _plan_commands(provider, complexity): + """Every rendered command that runs ``fluid plan`` (Jenkins: stage 6's body).""" + from fluid_build.forge.core.pipeline_templates import PipelineConfig, PipelineTemplateGenerator + + files = PipelineTemplateGenerator().generate_pipeline( + PipelineConfig(provider=provider, complexity=complexity) + ) + found = [] + for text in files.values(): + if provider.value == "jenkins": + start = text.index("stage('6 - plan')") + found.append(text[start : text.index("stage('7", start)]) + continue + for doc in yaml.safe_load_all(text): + found.extend(s for s in _strings(doc) if "fluid plan" in s) + return found + + +def _cases(): + from fluid_build.forge.core.pipeline_templates import PipelineComplexity, PipelineProvider + + return [(p, c) for p in PipelineProvider for c in PipelineComplexity] + + +@pytest.mark.parametrize( + "provider, complexity", _cases(), ids=lambda v: getattr(v, "value", str(v)) +) +def test_every_generated_plan_stage_checks_sovereignty(provider, complexity): + commands = _plan_commands(provider, complexity) + if provider.value == "tekton" and complexity.value == "basic": + # The basic Tekton pipeline references a `fluid-plan` Task it does not + # render; there is no plan command in it to carry the flag. + assert commands == [] + return + assert commands, "no plan command rendered" + for command in commands: + assert "--check-sovereignty" in command, command + + +# ── --env for a declared environment with no overlay is refused ─────────── + +_SILVER = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "silver.subscription_status_summary", + "name": "Subscription Status Summary", + "description": "Declared-env fixture.", + "domain": "Customer", + "metadata": {"layer": "Silver", "owner": {"team": "data-platform", "email": "dp@example.com"}}, + "exposes": [ + { + "exposeId": "summary", + "kind": "table", + "binding": { + "platform": "local", + "format": "parquet", + "location": {"path": "o.parquet"}, + }, + "contract": {"schema": [{"name": "status", "type": "STRING"}]}, + } + ], +} + + +@pytest.fixture +def silver(tmp_path, monkeypatch): + """fluid-demo-env's layout: a workspace declaring each product's targets, + and a product with an aws overlay but (yet) no gcp one.""" + from fluid_build import loader + + loader._NOTED_MISSING_OVERLAYS.clear() + (tmp_path / "fluid.workspace.yaml").write_text( + yaml.safe_dump( + { + "workspace": {"name": "demo"}, + "expected-environments": {"subscription_status_summary": ["local", "aws", "gcp"]}, + } + ), + encoding="utf-8", + ) + product = tmp_path / "contracts" / "subscription_status_summary" + (product / "overlays").mkdir(parents=True) + (product / "overlays" / "aws.yaml").write_text( + yaml.safe_dump({"exposes": [{"binding": {"platform": "aws", "format": "parquet"}}]}), + encoding="utf-8", + ) + contract = product / "contract.fluid.yaml" + contract.write_text(yaml.safe_dump(_SILVER, sort_keys=False), encoding="utf-8") + monkeypatch.chdir(product) + return contract + + +def test_a_declared_env_with_no_overlay_is_refused(silver): + from fluid_build._contract_loader import CLIError + from fluid_build.loader import load_with_overlay + + with pytest.raises(CLIError) as exc: + load_with_overlay(silver, "gcp") + assert exc.value.event == "overlay_declared_but_missing" + assert exc.value.context["declared_by"] == ( + "fluid.workspace.yaml expected-environments (subscription_status_summary)" + ) + assert "bound to local" in str(exc.value) + + +def test_validate_plan_and_bundle_all_refuse_it(silver, tmp_path): + from fluid_build.cli import main + + assert main(["validate", str(silver), "--env", "gcp"]) == 1 + assert main(["plan", str(silver), "--env", "gcp", "--out", str(tmp_path / "p.json")]) == 1 + assert not (tmp_path / "p.json").exists() + assert main(["bundle", str(silver), "--env", "gcp", "--out", str(tmp_path / "b.tgz")]) != 0 + + +def test_the_env_the_base_is_bound_to_and_dev_are_the_base(silver): + from fluid_build.loader import load_with_overlay + + assert load_with_overlay(silver, "local")["exposes"][0]["binding"]["platform"] == "local" + assert load_with_overlay(silver, "dev")["exposes"][0]["binding"]["platform"] == "local" + assert load_with_overlay(silver, "aws")["exposes"][0]["binding"]["platform"] == "aws" + + +def test_an_undeclared_env_still_warns_and_says_what_the_base_binds_to(silver, caplog): + from fluid_build.loader import load_with_overlay + + caplog.set_level(logging.WARNING, logger="fluid.loader") + load_with_overlay(silver, "prod") + (record,) = [r for r in caplog.records if "overlay_not_found" in r.getMessage()] + assert "it binds to local, not to 'prod'" in record.getMessage() + + +def test_the_contract_s_own_environments_block_declares_too(silver): + from fluid_build._contract_loader import CLIError + from fluid_build.loader import load_with_overlay + + doc = dict(_SILVER, environments={"staging": {}}) + silver.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + with pytest.raises(CLIError) as exc: + load_with_overlay(silver, "staging") + assert exc.value.context["declared_by"] == "the contract's environments block" diff --git a/tests/cli/test_policy_apply_provider_message.py b/tests/cli/test_policy_apply_provider_message.py new file mode 100644 index 00000000..9d2954d6 --- /dev/null +++ b/tests/cli/test_policy_apply_provider_message.py @@ -0,0 +1,139 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stage 8 (``fluid policy-apply``) survives a provider result that says ``message``. + +``policy_apply.run`` logs the provider's result as the event payload, +``info(logger, "policy_apply_result", **res)``. The GCP provider's result +carries a ``message`` key, which collided with ``info()``'s own ``message`` +parameter: ``TypeError: info() got multiple values for argument 'message'``, +exit 1, on every generated gcp pipeline (measured on 0.16.5 for all three +demo products). The bindings file below is the one ``fluid policy-compile`` +wrote for the demo's bronze gcp overlay. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +import pytest + +from fluid_build.cli import _logging +from fluid_build.cli.policy_apply import run + +pytestmark = pytest.mark.unit + +_GCP_BINDINGS = { + "bindings": [ + { + "dataset": "demo_bronze", + "principal": "group:data-platform@northwind.example", + "project": "northwind-demo", + "provider": "gcp", + "resource_id": "northwind-demo.demo_bronze", + "resource_type": "bigquery.dataset", + "roles": ["roles/bigquery.dataViewer"], + } + ], + "warnings": [], +} + + +def _args(path: Path, mode: str = "enforce") -> argparse.Namespace: + return argparse.Namespace(bindings=str(path), mode=mode, provider=None, project=None) + + +@pytest.fixture +def events(caplog): + caplog.set_level(logging.DEBUG, logger="test.policy_apply_message") + return caplog + + +def _payloads(caplog, name: str): + out = [] + for record in caplog.records: + try: + doc = json.loads(record.getMessage()) + except ValueError: + continue + if doc.get("message") == name: + out.append(doc) + return out + + +@pytest.mark.parametrize("mode", ["check", "enforce"]) +def test_the_gcp_provider_result_is_logged_not_a_type_error(tmp_path, events, monkeypatch, mode): + for var in ("GOOGLE_CLOUD_PROJECT", "FLUID_PROJECT", "GCLOUD_PROJECT", "FLUID_PROVIDER"): + monkeypatch.delenv(var, raising=False) + bindings = tmp_path / "bindings.json" + bindings.write_text(json.dumps(_GCP_BINDINGS), encoding="utf-8") + + rc = run(_args(bindings, mode), logging.getLogger("test.policy_apply_message")) + + assert rc == 0 + (event,) = _payloads(events, "policy_apply_result") + # The event keeps its name; the provider's own sentence is kept, renamed. + assert event["status"] == "ok" + assert "declaratively" in event["extra_message"] + assert event["bindings"] == 1 + + +class _Talkative: + """A provider whose result names every envelope key.""" + + name = "talkative" + + def apply_policy(self, data, mode="check"): + return { + "status": "ok", + "message": "provider sentence", + "time": "provider time", + "level": "provider level", + "name": "provider name", + } + + +def test_any_provider_result_that_names_an_envelope_key_is_kept(tmp_path, events, monkeypatch): + from fluid_build.cli import policy_apply + + monkeypatch.setattr(policy_apply, "build_provider", lambda *a, **k: _Talkative()) + bindings = tmp_path / "bindings.json" + bindings.write_text( + json.dumps({"bindings": [{"provider": "talkative", "roles": ["r"]}]}), encoding="utf-8" + ) + + assert run(_args(bindings), logging.getLogger("test.policy_apply_message")) == 0 + + (event,) = _payloads(events, "policy_apply_result") + assert event["level"] == "INFO" + assert event["name"] == "fluid.cli" + assert event["extra_message"] == "provider sentence" + assert event["extra_time"] == "provider time" + assert event["extra_level"] == "provider level" + assert event["extra_name"] == "provider name" + + +@pytest.mark.parametrize("helper", ["info", "warn", "error"]) +def test_every_structured_helper_takes_a_message_key(helper, caplog): + caplog.set_level(logging.DEBUG, logger="test.policy_apply_message.helpers") + log = logging.getLogger("test.policy_apply_message.helpers") + getattr(_logging, helper)(log, "the_event", message="payload", logger="also payload") + (record,) = caplog.records + doc = json.loads(record.getMessage()) + assert doc["message"] == "the_event" + assert doc["extra_message"] == "payload" + assert doc["logger"] == "also payload" diff --git a/tests/forge/test_artifact_fanout_schedule.py b/tests/forge/test_artifact_fanout_schedule.py index cc395aca..f65c77b8 100644 --- a/tests/forge/test_artifact_fanout_schedule.py +++ b/tests/forge/test_artifact_fanout_schedule.py @@ -46,6 +46,9 @@ LOG = logging.getLogger("test.artifact_fanout_schedule") DAG_REL = "schedule/bronze.customer_subscriptions/ingest_subscriptions_dag.py" +#: An env's DAGs get a directory of their own (``__``), so an aws +#: and a gcp pipeline syncing to one Airflow never delete each other's DAG. +DAG_REL_AWS = "schedule/bronze.customer_subscriptions__aws/ingest_subscriptions_dag.py" #: An aws overlay that also moves the schedule, so the DAG shows which #: contract it was rendered from. @@ -140,7 +143,7 @@ def test_a_bundle_takes_its_env_and_contract_path_from_the_flags( ) == 0 ) - dag = tmp_path / "dist" / "artifacts" / DAG_REL + dag = tmp_path / "dist" / "artifacts" / DAG_REL_AWS loaded = load_dag(dag.read_text(), monkeypatch) assert loaded.namespace["FLUID_ENV_NAME"] == "aws" assert loaded.namespace["CONTRACT_PATH"] == DEMO_CONTRACT_PATH @@ -170,7 +173,7 @@ def test_a_raw_contract_is_rendered_with_its_env_overlay( ) argv = (DEMO_CONTRACT_PATH, "--out", "dist/artifacts", "--env", "aws") assert _stage3(tmp_path, monkeypatch, *argv) == 0 - loaded = load_dag((tmp_path / "dist" / "artifacts" / DAG_REL).read_text(), monkeypatch) + loaded = load_dag((tmp_path / "dist" / "artifacts" / DAG_REL_AWS).read_text(), monkeypatch) assert loaded.dag["schedule"] == "30 1 * * *" assert loaded.namespace["FLUID_ENV_NAME"] == "aws" # Fanned out through a temporary --env bundle, the DAG still runs the @@ -326,7 +329,7 @@ def test_without_env_stage_3_uses_fluid_env( ) monkeypatch.setenv("FLUID_ENV", "aws") assert _stage3(tmp_path, monkeypatch, DEMO_CONTRACT_PATH, "--out", "dist/artifacts") == 0 - loaded = load_dag((tmp_path / "dist" / "artifacts" / DAG_REL).read_text(), monkeypatch) + loaded = load_dag((tmp_path / "dist" / "artifacts" / DAG_REL_AWS).read_text(), monkeypatch) assert loaded.namespace["FLUID_ENV_NAME"] == "aws" assert loaded.dag["schedule"] == "30 1 * * *" @@ -337,7 +340,7 @@ def test_the_flag_beats_fluid_env( monkeypatch.setenv("FLUID_ENV", "gcp") argv = (DEMO_CONTRACT_PATH, "--out", "dist/artifacts", "--env", "aws") assert _stage3(tmp_path, monkeypatch, *argv) == 0 - loaded = load_dag((tmp_path / "dist" / "artifacts" / DAG_REL).read_text(), monkeypatch) + loaded = load_dag((tmp_path / "dist" / "artifacts" / DAG_REL_AWS).read_text(), monkeypatch) assert loaded.namespace["FLUID_ENV_NAME"] == "aws" @pytest.mark.parametrize("fluid_env", [None, ""]) diff --git a/tests/iac/test_iac_state_key_migration_moto.py b/tests/iac/test_iac_state_key_migration_moto.py new file mode 100644 index 00000000..339088e8 --- /dev/null +++ b/tests/iac/test_iac_state_key_migration_moto.py @@ -0,0 +1,357 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The provider-keyed state default, and the move of the old state, against moto. + +A bucket-only ``FLUID_STATE_BACKEND`` used to key a contract's state by its +id alone, ``fluid//terraform.tfstate``, so the aws and the gcp apply of +one contract (two ``--env`` overlays) shared one state and each plan read the +other cloud's resources as orphans to destroy. The default is now +``fluid///terraform.tfstate``, and the first apply after the +upgrade moves the old state there with OpenTofu's own ``init +-migrate-state``. + +Everything here is real ``tofu`` against a moto S3 (the state bucket) and +moto AWS APIs (the product's resources): the old release's apply is played +by an apply with the old key spelled out, which is exactly the object that +release wrote. Pinned: the move, a plan after it that changes nothing, the +old object left in place, a second apply that moves nothing, a wiped +workdir (CI) as well as a kept one, the data-loss gate still closed after +the move, another provider's state at the old key left alone, a state of +two clouds refused, and a new key that already holds state never +overwritten. + +Skipped unless ``tofu`` is on PATH and moto's ``server`` extra is installed. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import logging +import shutil +from pathlib import Path +from typing import Any, Dict, Iterator + +import pytest +import yaml + +from fluid_build.cli import _apply_opentofu_engine as engine +from fluid_build.cli._common import CLIError +from fluid_build.iac import runner +from fluid_build.iac import state_migration as mig +from fluid_build.iac.credentials import build_tofu_env + +pytestmark = [pytest.mark.integration, pytest.mark.provider, pytest.mark.aws] + +_LOG = logging.getLogger("test.iac.state_key_migration") +_REGION = "us-east-1" +_CID = "bronze.customer_subscriptions" +_STATE_BUCKET = "fluid-state-migration" +_DATA_BUCKET = "migration-moto-lake" +_LEGACY_KEY = f"fluid/{_CID}/terraform.tfstate" +_AWS_KEY = f"fluid/{_CID}/aws/terraform.tfstate" +_GCP_KEY = f"fluid/{_CID}/gcp/terraform.tfstate" + + +def _have_moto_server() -> bool: + try: + from moto.server import ThreadedMotoServer # noqa: F401 + + return True + except Exception: # noqa: BLE001 + return False + + +pytestmark.append( + pytest.mark.skipif( + runner.tofu_path() is None or not _have_moto_server(), + reason="needs `tofu` on PATH + moto server extra (pip install 'moto[server]')", + ) +) + + +@pytest.fixture(scope="module") +def plugin_cache(tmp_path_factory: pytest.TempPathFactory) -> str: + """One provider download for the module, not one per workdir.""" + return str(tmp_path_factory.mktemp("tofu-plugin-cache")) + + +@pytest.fixture +def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: + """A fresh moto server holding the state bucket; tofu and boto3 aim at it.""" + import requests + from moto.server import ThreadedMotoServer + + server = ThreadedMotoServer(port=0, verbose=False) + server.start() + try: + _, port = server.get_host_and_port() + endpoint = f"http://127.0.0.1:{port}" + with contextlib.suppress(Exception): + requests.post(f"{endpoint}/moto-api/reset", timeout=5) + for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "FLUID_STATE_BACKEND", "FLUID_PROVIDER"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") # pragma: allowlist secret + monkeypatch.setenv("AWS_REGION", _REGION) + monkeypatch.setenv("AWS_DEFAULT_REGION", _REGION) + monkeypatch.setenv("AWS_CONFIG_FILE", "/dev/null") + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "/dev/null") + monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true") + monkeypatch.setenv("TF_PLUGIN_CACHE_DIR", plugin_cache) + monkeypatch.chdir(tmp_path) + _s3(endpoint).create_bucket(Bucket=_STATE_BUCKET) + yield endpoint + finally: + server.stop() + + +def _s3(endpoint: str): + import boto3 + + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id="testing", + aws_secret_access_key="testing", # pragma: allowlist secret + region_name=_REGION, + ) + + +def _object(endpoint: str, key: str) -> Dict[str, Any]: + body = _s3(endpoint).get_object(Bucket=_STATE_BUCKET, Key=key)["Body"].read() + return json.loads(body) + + +def _keys(endpoint: str) -> set: + listing = _s3(endpoint).list_objects_v2(Bucket=_STATE_BUCKET) + return {o["Key"] for o in listing.get("Contents", [])} + + +def _contract(*, table: str = "customer_subscriptions") -> Dict[str, Any]: + exposes = [ + { + "exposeId": "subscriptions", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": { + "database": "demo_bronze", + "table": table, + "bucket": _DATA_BUCKET, + "path": "bronze/customer_subscriptions/", + }, + }, + "contract": { + "schema": [ + {"name": "subscription_id", "type": "VARCHAR", "required": True}, + {"name": "msisdn", "type": "VARCHAR"}, + ] + }, + } + ] + return { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": _CID, + "name": "Customer Subscriptions", + "domain": "Customer", + "metadata": {"layer": "Bronze", "owner": {"team": "data-platform"}}, + "exposes": exposes, + } + + +def _write(root: Path, contract: Dict[str, Any]) -> Path: + path = root / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + return path + + +def _apply(contract_path: Path, root: Path, *, state_backend=None, dry_run=False) -> None: + args = argparse.Namespace( + contract=str(contract_path), + env=None, + provider=None, + workspace_dir=str(root), + state_backend=state_backend, + dry_run=dry_run, + allow_data_loss=False, + no_verify_plan_binding=True, + ) + assert engine.apply_via_opentofu(args, _LOG) == 0 + + +def _apply_as_the_old_release(contract_path: Path, root: Path) -> None: + """The object a bucket-only FLUID_STATE_BACKEND wrote before this change.""" + _apply(contract_path, root, state_backend=f"s3://{_STATE_BUCKET}/{_LEGACY_KEY}") + + +def _upgraded_plan(contract_path: Path, root: Path, monkeypatch) -> None: + monkeypatch.setenv("FLUID_STATE_BACKEND", f"s3://{_STATE_BUCKET}") + _apply(contract_path, root, dry_run=True) + + +def _block(key: str) -> Dict[str, Any]: + return {"s3": {"bucket": _STATE_BUCKET, "key": key}} + + +def _reconcile(tmp_path: Path, key: str, provider: str, *, migrate: bool): + """``reconcile_state_key`` from a workdir initialised on ``key``, as the apply calls it.""" + workdir = tmp_path / f"workdir-{provider}" + workdir.mkdir(exist_ok=True) + (workdir / "main.tf.json").write_text( + json.dumps({"terraform": {"backend": _block(key)}}), encoding="utf-8" + ) + env = build_tofu_env() + assert runner.tofu_init(str(workdir), env=env, reconfigure=True).ok + return mig.reconcile_state_key( + workdir=workdir, + current=_block(key), + legacy=_block(_LEGACY_KEY), + provider=provider, + env=env, + migrate=migrate, + ) + + +@pytest.mark.parametrize("wipe_workdir", [False, True], ids=["kept-workdir", "wiped-workdir"]) +def test_the_old_state_moves_and_the_plan_after_it_changes_nothing( + moto, tmp_path, monkeypatch, capsys, wipe_workdir +): + contract = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract, tmp_path) + old = _object(moto, _LEGACY_KEY) + assert old["resources"], "the old release's apply wrote no resources" + assert _AWS_KEY not in _keys(moto) + if wipe_workdir: + # A CI workspace is wiped after every run: no .terraform/ records the + # old key, and the move still has to happen. + shutil.rmtree(tmp_path / ".fluid") + capsys.readouterr() + + _upgraded_plan(contract, tmp_path, monkeypatch) + + # The console wraps long lines; the checks read the text unwrapped. + printed = capsys.readouterr().out.replace("\n", "") + assert f"remote: s3://{_STATE_BUCKET}/{_AWS_KEY} (from FLUID_STATE_BACKEND)" in printed + assert f"state move: moved {len(old['resources'])} resource(s)" in printed + assert "tofu plan: +0 ~0 -0" in printed + moved = _object(moto, _AWS_KEY) + # OpenTofu writes the copy under a fresh lineage; the resources are the same. + assert moved["resources"] == old["resources"] + # Never lost: the old object is left exactly where it was. + assert _object(moto, _LEGACY_KEY) == old + + # The next apply finds its state at the new key and moves nothing. + _upgraded_plan(contract, tmp_path, monkeypatch) + again = capsys.readouterr().out.replace("\n", "") + assert "state move:" not in again + assert "tofu plan: +0 ~0 -0" in again + + +def test_the_data_loss_gate_still_closes_after_the_move(moto, tmp_path, monkeypatch): + """The moved state is the one the gate judges: renaming the table plans a + destroy, and without --allow-data-loss the apply refuses it.""" + contract = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract, tmp_path) + monkeypatch.setenv("FLUID_STATE_BACKEND", f"s3://{_STATE_BUCKET}") + _write(tmp_path, _contract(table="customer_subscriptions_v2")) + args = argparse.Namespace( + contract=str(contract), + env=None, + provider=None, + workspace_dir=str(tmp_path), + state_backend=None, + dry_run=False, + allow_data_loss=False, + no_verify_plan_binding=True, + ) + with pytest.raises(CLIError) as exc: + engine.apply_via_opentofu(args, _LOG) + assert exc.value.event == "opentofu_data_loss_gate" + assert _AWS_KEY in _keys(moto) + + +def test_another_providers_state_at_the_old_key_is_left_alone(moto, tmp_path): + """The gcp apply of a contract whose aws state still sits at the old key: + it is the aws apply's to move, not the gcp apply's.""" + contract = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract, tmp_path) + old = _object(moto, _LEGACY_KEY) + + outcome = _reconcile(tmp_path, _GCP_KEY, "gcp", migrate=True) + + assert outcome.outcome == mig.OTHER_PROVIDER + assert "aws" in outcome.detail + assert _GCP_KEY not in _keys(moto) + assert _object(moto, _LEGACY_KEY) == old + + +def test_a_state_holding_two_clouds_is_refused_and_nothing_moves(moto, tmp_path): + contract = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract, tmp_path) + old = _object(moto, _LEGACY_KEY) + mixed = dict(old) + mixed["resources"] = list(old["resources"]) + [ + { + "mode": "managed", + "type": "google_bigquery_dataset", + "name": "bronze_customer_subscriptions_demo_bronze", + "provider": 'provider["registry.opentofu.org/hashicorp/google"]', + "instances": [], + } + ] + _s3(moto).put_object( + Bucket=_STATE_BUCKET, Key=_LEGACY_KEY, Body=json.dumps(mixed).encode("utf-8") + ) + + with pytest.raises(mig.StateMigrationError) as exc: + _reconcile(tmp_path, _AWS_KEY, "aws", migrate=True) + + assert exc.value.code == "state_migration_ambiguous" + assert "aws, gcp" in str(exc.value) + assert _AWS_KEY not in _keys(moto) + + +def test_a_new_key_that_holds_state_is_never_overwritten(moto, tmp_path, monkeypatch): + contract = _write(tmp_path, _contract()) + # This provider already applied at the new key... + monkeypatch.setenv("FLUID_STATE_BACKEND", f"s3://{_STATE_BUCKET}") + _apply(contract, tmp_path) + current = _object(moto, _AWS_KEY) + # ...and an older state of this provider still sits at the old one. + older = dict(current, lineage="00000000-0000-0000-0000-000000000000") + _s3(moto).put_object( + Bucket=_STATE_BUCKET, Key=_LEGACY_KEY, Body=json.dumps(older).encode("utf-8") + ) + + outcome = _reconcile(tmp_path, _AWS_KEY, "aws", migrate=True) + + assert outcome.outcome == mig.CURRENT + assert _object(moto, _AWS_KEY) == current + + +def test_a_read_only_caller_reads_the_old_key_until_the_apply_moves_it(moto, tmp_path): + contract = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract, tmp_path) + + outcome = _reconcile(tmp_path, _AWS_KEY, "aws", migrate=False) + + assert outcome.outcome == mig.PENDING + assert outcome.read_from == _block(_LEGACY_KEY) + assert _AWS_KEY not in _keys(moto) diff --git a/tests/iac/test_state_backend_env_default.py b/tests/iac/test_state_backend_env_default.py index c57e8d2f..eb8fe546 100644 --- a/tests/iac/test_state_backend_env_default.py +++ b/tests/iac/test_state_backend_env_default.py @@ -35,6 +35,7 @@ from fluid_build.cli import _apply_opentofu_engine as engine from fluid_build.cli._common import CLIError +from fluid_build.iac.state_migration import CURRENT, StateReconciliation pytestmark = [pytest.mark.unit] @@ -84,6 +85,13 @@ def _init_stops_here(*_a: Any, **_k: Any) -> SimpleNamespace: return SimpleNamespace(ok=False, stderr="stub: tofu init not run", stdout="") monkeypatch.setattr(engine.runner, "tofu_init", _init_stops_here) + # The move of a pre-provider-key state (``iac.state_migration``) runs its + # own ``tofu init``; it is not under test here and finds nothing to move. + monkeypatch.setattr( + engine, + "_reconcile_state", + lambda **kw: StateReconciliation(CURRENT, kw["legacy"], kw["current"]), + ) args = argparse.Namespace( contract=str(contract), env=None, @@ -187,9 +195,9 @@ def test_a_bucket_only_env_value_gives_each_contract_its_own_s3_key( second, _ = _apply_and_read_backend( tmp_path / "b", monkeypatch, flag=None, contract_text=_OTHER_CONTRACT ) - assert first == {"s3": {"bucket": "ci-state", "key": "fluid/demo.state/terraform.tfstate"}} + assert first == {"s3": {"bucket": "ci-state", "key": "fluid/demo.state/aws/terraform.tfstate"}} assert second == { - "s3": {"bucket": "ci-state", "key": "fluid/demo.other_state/terraform.tfstate"} + "s3": {"bucket": "ci-state", "key": "fluid/demo.other_state/aws/terraform.tfstate"} } @@ -199,7 +207,7 @@ def test_a_bucket_only_env_value_gives_each_contract_its_own_gcs_prefix( ) -> None: monkeypatch.setenv("FLUID_STATE_BACKEND", spec) backend, _ = _apply_and_read_backend(tmp_path, monkeypatch, flag=None) - assert backend == {"gcs": {"bucket": "ci-state", "prefix": "fluid/demo.state"}} + assert backend == {"gcs": {"bucket": "ci-state", "prefix": "fluid/demo.state/aws"}} def test_the_flag_keeps_the_shared_legacy_key_for_a_contract_without_packaging( @@ -227,8 +235,8 @@ def test_ids_the_old_key_folded_together_get_their_own_state( flag=None, contract_text=_CONTRACT.replace("id: demo.state", "id: demo_state"), ) - assert dotted["s3"]["key"] == "fluid/demo.state/terraform.tfstate" - assert underscored["s3"]["key"] == "fluid/demo_state/terraform.tfstate" + assert dotted["s3"]["key"] == "fluid/demo.state/aws/terraform.tfstate" + assert underscored["s3"]["key"] == "fluid/demo_state/aws/terraform.tfstate" def test_an_id_that_cannot_key_a_state_is_a_typed_error_naming_the_variable( @@ -278,7 +286,7 @@ def test_the_state_line_names_the_object_each_form_resolved_to( " state: remote: s3://ci-state/fluid/terraform.tfstate (from --state-backend)" ] assert env_lines == [ - " state: remote: s3://ci-state/fluid/demo.state/terraform.tfstate" + " state: remote: s3://ci-state/fluid/demo.state/aws/terraform.tfstate" " (from FLUID_STATE_BACKEND)" ] diff --git a/tests/iac/test_state_key_per_provider.py b/tests/iac/test_state_key_per_provider.py new file mode 100644 index 00000000..0cb7cc25 --- /dev/null +++ b/tests/iac/test_state_key_per_provider.py @@ -0,0 +1,190 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The default remote state key names the provider. + +Measured on 0.16.5 with ``FLUID_STATE_BACKEND=s3://fluid-demo-lab-state-…``: +``resolve_state_target`` gave the aws and the gcp apply of +``bronze.customer_subscriptions`` the same key, +``fluid/bronze.customer_subscriptions/terraform.tfstate``, so the gcp plan +would have read the aws resources as orphans to destroy. Offline unit pins +for the key, the old key the move reads from, and how a state is attributed +to a provider; ``test_iac_state_key_migration_moto.py`` runs the move itself +with real ``tofu``. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pytest + +from fluid_build.cli._apply_opentofu_engine import resolve_state_target +from fluid_build.iac import state_migration as mig +from fluid_build.iac.backend import default_state_key, legacy_default_backend, parse_backend + +pytestmark = pytest.mark.unit + +_CID = "bronze.customer_subscriptions" +_CONTRACT = {"id": _CID, "name": "Customer Subscriptions"} +_PACKAGED = {"id": _CID, "packaging": {"mode": "isolated"}} + + +def _args(tmp_path: Path, flag=None) -> argparse.Namespace: + return argparse.Namespace(state_backend=flag, workspace_dir=str(tmp_path)) + + +def test_the_aws_and_the_gcp_apply_of_one_contract_get_two_states(tmp_path, monkeypatch): + monkeypatch.setenv("FLUID_STATE_BACKEND", "s3://fluid-demo-lab-state-111111111111") + aws = resolve_state_target(_args(tmp_path), _CONTRACT, "aws") + gcp = resolve_state_target(_args(tmp_path), _CONTRACT, "gcp") + assert aws.backend == { + "s3": { + "bucket": "fluid-demo-lab-state-111111111111", + "key": f"fluid/{_CID}/aws/terraform.tfstate", + } + } + assert gcp.backend["s3"]["key"] == f"fluid/{_CID}/gcp/terraform.tfstate" + # ...and both know where the previous release kept the state. + legacy = { + "s3": { + "bucket": "fluid-demo-lab-state-111111111111", + "key": f"fluid/{_CID}/terraform.tfstate", + } + } + assert aws.legacy_backend == legacy + assert gcp.legacy_backend == legacy + + +def test_a_gcs_prefix_names_the_provider_too(tmp_path, monkeypatch): + monkeypatch.setenv("FLUID_STATE_BACKEND", "gcs://team-state") + target = resolve_state_target(_args(tmp_path), _CONTRACT, "gcp") + assert target.backend == {"gcs": {"bucket": "team-state", "prefix": f"fluid/{_CID}/gcp"}} + assert target.legacy_backend == {"gcs": {"bucket": "team-state", "prefix": f"fluid/{_CID}"}} + + +def test_a_packaging_contract_on_the_flag_gets_the_provider_segment(tmp_path, monkeypatch): + monkeypatch.delenv("FLUID_STATE_BACKEND", raising=False) + target = resolve_state_target(_args(tmp_path, "s3://ci-state"), _PACKAGED, "aws") + assert ( + target.backend["s3"]["key"] == "fluid/bronze_customer_subscriptions/aws/terraform.tfstate" + ) + assert target.legacy_backend["s3"]["key"] == ( + "fluid/bronze_customer_subscriptions/terraform.tfstate" + ) + + +@pytest.mark.parametrize( + "flag", + ["s3://ci-state", "s3://ci-state/team/explicit.tfstate", "gcs://ci-state/team/x", ""], + ids=["legacy-shared-key", "explicit-key", "explicit-prefix", "local"], +) +def test_keys_nobody_defaulted_are_never_moved(tmp_path, monkeypatch, flag): + """The shared legacy key, an explicit key or prefix and local state keep + their location, so there is nothing to migrate from.""" + monkeypatch.delenv("FLUID_STATE_BACKEND", raising=False) + target = resolve_state_target(_args(tmp_path, flag), _CONTRACT, "aws") + assert target.legacy_backend is None + if flag == "s3://ci-state": + assert target.backend == {"s3": {"bucket": "ci-state", "key": "fluid/terraform.tfstate"}} + + +def test_parse_backend_without_a_provider_is_unchanged(): + """Callers that do not name a provider keep the old keys byte for byte.""" + assert parse_backend("s3://b", _CONTRACT, per_contract_default=True) == { + "s3": {"bucket": "b", "key": f"fluid/{_CID}/terraform.tfstate"} + } + assert default_state_key(_CONTRACT) == "fluid/terraform.tfstate" + + +def test_a_provider_that_is_not_one_key_segment_is_refused(): + with pytest.raises(ValueError, match="cannot name a state key segment"): + default_state_key(_CONTRACT, per_contract=True, provider="../aws") + with pytest.raises(ValueError): + legacy_default_backend("s3://b", _CONTRACT, per_contract_default=True, provider="a/b") + + +# ── Whose state is it? ──────────────────────────────────────────────────── + + +def _res(source: str, rtype: str = "x") -> dict: + return {"type": rtype, "provider": f'provider["registry.opentofu.org/{source}"]'} + + +@pytest.mark.parametrize( + "resources, provider, verdict", + [ + ([_res("hashicorp/aws"), _res("hashicorp/null")], "aws", "mine"), + ([_res("hashicorp/google")], "gcp", "mine"), + ([_res("hashicorp/aws")], "gcp", "other"), + ([_res("hashicorp/google")], "aws", "other"), + ([_res("hashicorp/aws"), _res("hashicorp/google")], "aws", "ambiguous"), + ([_res("hashicorp/aws"), _res("hashicorp/google")], "gcp", "ambiguous"), + ([_res("hashicorp/random")], "aws", "ambiguous"), + ([{"type": "x", "provider": "not an address"}], "aws", "ambiguous"), + ([_res("snowflake-labs/snowflake")], "snowflake", "mine"), + ], + ids=[ + "aws-own", + "gcp-own", + "aws-state-seen-by-gcp", + "gcp-state-seen-by-aws", + "two-clouds-aws", + "two-clouds-gcp", + "unknown-provider", + "unreadable-address", + "snowflake-pre-v2-source", + ], +) +def test_a_state_is_attributed_by_its_resources_providers(resources, provider, verdict): + assert mig.classify(resources, provider)[0] == verdict + + +def test_a_terraform_written_state_is_read_the_same(): + resources = [{"provider": 'provider["registry.terraform.io/hashicorp/aws"].west'}] + assert mig.classify(resources, "aws")[0] == "mine" + + +def test_state_pull_output_is_parsed_and_an_absent_state_is_empty(): + assert not mig.parse_state("").exists + empty = '{"version":4,"serial":0,"lineage":"","resources":[]}' + assert not mig.parse_state(empty).exists + doc = mig.parse_state(json.dumps({"lineage": "L", "serial": 3, "resources": [_res("a/b")]})) + assert (doc.exists, doc.serial, len(doc.resources)) == (True, 3, 1) + with pytest.raises(mig.StateMigrationError): + mig.parse_state("not json") + + +def test_the_workdir_s_recorded_backend_is_compared_on_the_fields_forge_writes(tmp_path): + (tmp_path / ".terraform").mkdir() + (tmp_path / ".terraform" / "terraform.tfstate").write_text( + json.dumps( + { + "backend": { + "type": "s3", + "config": {"bucket": "b", "key": "fluid/x/terraform.tfstate", "region": None}, + } + } + ), + encoding="utf-8", + ) + assert mig.records_backend( + tmp_path, {"s3": {"bucket": "b", "key": "fluid/x/terraform.tfstate"}} + ) + assert not mig.records_backend( + tmp_path, {"s3": {"bucket": "b", "key": "fluid/x/aws/terraform.tfstate"}} + ) + assert not mig.records_backend(tmp_path / "nowhere", {"s3": {"bucket": "b"}}) diff --git a/tests/providers/test_gcp_sovereignty_fail_closed.py b/tests/providers/test_gcp_sovereignty_fail_closed.py new file mode 100644 index 00000000..b8fb54ee --- /dev/null +++ b/tests/providers/test_gcp_sovereignty_fail_closed.py @@ -0,0 +1,333 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sovereignty fails closed on GCP. + +Measured on 0.16.5 against the demo's bronze contract (``jurisdiction: EU``, +``allowedRegions: [eu-north-1, eu-west-1, europe-west1]``, strict): + +* a gcp overlay with no region validated (rc 0), ``plan --check-sovereignty`` + printed PASS, and the emitted dataset landed in ``US``; +* ``us-central1`` was refused by ``fluid validate`` only: ``fluid generate + iac`` emitted it, rc 0, because the GCP provider had no sovereignty hook; +* a region given as ``location.location`` (which the GCP emitter reads) was + never checked at all. + +Each is pinned here against the real CLI entry points, the GCP IaC plugin +and the GCP provider. No cloud is called: the plugin and the provider only +compute, and the BigQuery client is faked. +""" + +from __future__ import annotations + +import copy +import logging +from pathlib import Path +from typing import Any, Dict, Optional + +import pytest +import yaml + +from fluid_build._errors import SovereigntyViolationError +from fluid_build.iac import get_iac_plugin +from fluid_build.policy.sovereignty import SovereigntyValidator, binding_region + +pytestmark = pytest.mark.unit + +_LOG = logging.getLogger("test.gcp_sovereignty") + +_SOVEREIGNTY = { + "jurisdiction": "EU", + "allowedRegions": ["eu-north-1", "eu-west-1", "europe-west1"], + "deniedRegions": ["us-east-1", "us-west-2"], + "dataResidency": True, + "crossBorderTransfer": False, + "enforcementMode": "strict", +} + + +def _contract( + location: Dict[str, Any], + *, + mode: Optional[str] = "strict", + platform: str = "gcp", + sovereignty: bool = True, +) -> Dict[str, Any]: + doc: Dict[str, Any] = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "bronze.customer_subscriptions", + "name": "Customer Subscriptions", + "description": "Sovereignty fixture.", + "domain": "Customer", + "metadata": { + "layer": "Bronze", + "owner": {"team": "data-platform", "email": "dp@example.com"}, + }, + "exposes": [ + { + "exposeId": "subscriptions", + "kind": "table", + "binding": { + "platform": platform, + "format": "bigquery_table" if platform == "gcp" else "parquet", + "location": location, + }, + "contract": {"schema": [{"name": "subscription_id", "type": "STRING"}]}, + } + ], + } + if sovereignty: + doc["sovereignty"] = dict(_SOVEREIGNTY, enforcementMode=mode) + return doc + + +_BQ = {"project": "northwind-demo", "dataset": "demo_bronze", "table": "customer_subscriptions"} + + +def _findings(contract: Dict[str, Any]): + return SovereigntyValidator().validate(contract) + + +# ── The policy engine: fluid validate and plan --check-sovereignty ──────── + + +def test_a_gcp_binding_with_no_region_is_refused_under_strict(): + ok, violations = _findings(_contract(dict(_BQ))) + assert ok is False + (v,) = violations + assert v.severity == "error" + assert "declares no region" in v.message + assert v.expose_id == "subscriptions" + + +@pytest.mark.parametrize("mode, severity", [("advisory", "warning"), ("audit", "info")]) +def test_the_mode_decides_like_it_does_for_every_other_check(mode, severity): + ok, violations = _findings(_contract(dict(_BQ), mode=mode)) + assert ok is True + assert [v.severity for v in violations] == [severity] + + +def test_a_local_binding_with_no_region_is_still_not_a_finding(): + """The demo's local base carries the same sovereignty block.""" + ok, violations = _findings(_contract({"path": "data/x.parquet"}, platform="local")) + assert (ok, violations) == (True, []) + + +def test_an_aws_binding_with_no_region_is_refused_too(): + loc = {"bucket": "b", "database": "d", "table": "t", "path": "p/"} + ok, _ = _findings(_contract(loc, platform="aws")) + assert ok is False + + +def test_a_region_given_as_location_location_is_the_one_checked(): + """The GCP emitter falls back to ``location.location``; so must the check.""" + contract = _contract(dict(_BQ, location="us-central1")) + assert binding_region(contract["exposes"][0]["binding"]) == "us-central1" + ok, violations = _findings(contract) + assert ok is False + assert any("us-central1" in v.message for v in violations) + + +def test_the_bigquery_us_multi_region_is_us_jurisdiction(): + contract = _contract(dict(_BQ, region="US")) + contract["sovereignty"].pop("allowedRegions") + ok, violations = _findings(contract) + assert ok is False + assert any("jurisdiction: US" in v.message for v in violations) + + +def test_an_allowed_gcp_region_passes(): + assert _findings(_contract(dict(_BQ, region="europe-west1"))) == (True, []) + + +# ── The GCP OpenTofu plugin: fluid apply and fluid generate iac ──────────── + + +def _emit(contract: Dict[str, Any]) -> Dict[str, Any]: + return get_iac_plugin("gcp").emit(contract, []) + + +def test_the_emitter_refuses_to_guess_a_location_under_a_strict_policy(): + with pytest.raises(SovereigntyViolationError) as exc: + _emit(_contract(dict(_BQ))) + assert "subscriptions: Binding declares no region" in exc.value.what + # The rendered panel is rich markup: no [..] that would be eaten. + assert "[subscriptions]" not in exc.value.what + + +def test_the_emitter_refuses_an_out_of_jurisdiction_region(): + with pytest.raises(SovereigntyViolationError) as exc: + _emit(_contract(dict(_BQ, region="us-central1"))) + assert "us-central1" in exc.value.what + + +def test_the_emitter_places_an_allowed_region(): + resources = _emit(_contract(dict(_BQ, region="europe-west1"))) + (dataset,) = resources["google_bigquery_dataset"].values() + assert dataset["location"] == "europe-west1" + + +def test_without_a_policy_the_old_us_default_is_unchanged(): + resources = _emit(_contract(dict(_BQ), sovereignty=False)) + (dataset,) = resources["google_bigquery_dataset"].values() + assert dataset["location"] == "US" + + +def test_under_advisory_the_us_default_is_emitted_and_said_out_loud(caplog): + caplog.set_level(logging.WARNING) + resources = _emit(_contract(dict(_BQ), mode="advisory")) + (dataset,) = resources["google_bigquery_dataset"].values() + assert dataset["location"] == "US" + said = " ".join(r.getMessage() for r in caplog.records) + assert "declares no region" in said + assert "Region 'US' not in allowed regions list" in said + + +# ── The GCP provider hook (native planner), the way AWS refuses ─────────── + + +def test_the_provider_refuses_and_generate_iac_sees_a_sovereignty_veto(monkeypatch): + from fluid_build.cli.generate_iac import _is_sovereignty_refusal + from fluid_build.providers.base import ProviderError + from fluid_build.providers.gcp.provider import GcpProvider + + provider = GcpProvider(project="northwind-demo", region="europe-west1") + with pytest.raises(ProviderError) as exc: + provider.plan(_contract(dict(_BQ, region="us-central1"))) + assert _is_sovereignty_refusal(exc.value) + + +def test_a_provider_default_region_is_checked_where_it_is_used(): + """``--region`` defaults to europe-west3 and the SDK to us-central1; a + planned resource that inherits the provider's region is checked there.""" + from fluid_build.providers.base import ProviderError + from fluid_build.providers.gcp.provider import GcpProvider + + provider = GcpProvider(project="northwind-demo", region="europe-west3") + contract = _contract(dict(_BQ, region="europe-west1")) + scheduled = [{"op": "scheduler.ensure_job", "id": "nightly", "location": provider.region}] + with pytest.raises(ProviderError) as exc: + provider._validate_sovereignty(contract, scheduled) + assert "nightly: Region 'europe-west3' not in allowed regions list" in str(exc.value) + # The same planned job in an allowed region passes. + provider._validate_sovereignty(contract, [dict(scheduled[0], location="europe-west1")]) + + +# ── Through the real CLI ─────────────────────────────────────────────────── + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch) -> Path: + for var in ("FLUID_PROVIDER", "FLUID_REGION", "GOOGLE_APPLICATION_CREDENTIALS"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _write(root: Path, contract: Dict[str, Any]) -> Path: + path = root / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + return path + + +def _cli(*argv: str) -> int: + from fluid_build.cli import main + + return main(list(argv)) + + +def test_validate_refuses_a_gcp_binding_with_no_region(workspace): + assert _cli("validate", str(_write(workspace, _contract(dict(_BQ))))) == 1 + + +def test_generate_iac_refuses_an_out_of_jurisdiction_region(workspace): + contract = _write(workspace, _contract(dict(_BQ, region="us-central1"))) + assert _cli("generate", "iac", str(contract), "--out", str(workspace / "iac")) != 0 + assert not (workspace / "iac" / "main.tf.json").exists() + + +def test_generate_iac_emits_an_allowed_region(workspace): + contract = _write(workspace, _contract(dict(_BQ, region="europe-west1"))) + assert _cli("generate", "iac", str(contract), "--out", str(workspace / "iac")) == 0 + assert '"europe-west1"' in (workspace / "iac" / "main.tf.json").read_text(encoding="utf-8") + + +def test_plan_check_sovereignty_blocks_a_gcp_binding_with_no_region(workspace, capsys): + contract = _write(workspace, _contract(dict(_BQ))) + rc = _cli("plan", str(contract), "--out", str(workspace / "plan.json"), "--check-sovereignty") + assert rc == 1 + assert "PASS" not in capsys.readouterr().out + + +# ── The BigQuery load: no guessed location ──────────────────────────────── + + +def test_a_load_with_no_region_runs_where_the_table_is(tmp_path, monkeypatch): + from fluid_build.build_runners import _bigquery_load + + binding = copy.deepcopy(_contract(dict(_BQ))["exposes"][0]["binding"]) + target = _bigquery_load.bigquery_load_target(binding, {"exposeId": "subscriptions"}) + assert target is not None and target["location"] is None + + calls = [] + + class _Job: + output_rows = 1 + job_id = "job-1" + + def result(self): + return None + + class _Table: + schema: list = [] + location = "europe-west1" + + class _Client: + project = "northwind-demo" + + def __init__(self, project=None): + pass + + def get_table(self, table_id): + return _Table() + + def load_table_from_file(self, fh, table_id, job_config=None, location=None): + calls.append(location) + return _Job() + + class _Module: + Client = _Client + + class LoadJobConfig: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + class SourceFormat: + PARQUET = "PARQUET" + + class WriteDisposition: + WRITE_APPEND = "WRITE_APPEND" + WRITE_TRUNCATE = "WRITE_TRUNCATE" + + monkeypatch.setattr(_bigquery_load, "_bigquery_module", lambda: _Module) + landed = tmp_path / "rows.parquet" + landed.write_bytes(b"PAR1") + mode = sorted(_bigquery_load._WRITE_DISPOSITION)[0] + sink = sorted(_bigquery_load._SOURCE_FORMAT)[0] + _bigquery_load.load_file( + str(landed), target, mode=mode, sink_format=sink, expected_rows=1, logger=_LOG + ) + assert calls == ["europe-west1"] From cc9784c424921dbe5460f6b40b164b34b18f4285 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 13:52:01 +0200 Subject: [PATCH 02/10] fix(platform): a dry-run never moves state, strict GCP sovereignty fails closed on an unplaceable location, and a Pub/Sub topic keeps its region Review round on the one-contract-two-clouds platform safety change. State. `fluid apply --dry-run` (the generated Jenkins default APPLY_MODE) ran the state move and wrote the new key; measured with real tofu against moto, the key set grew by fluid//aws/terraform.tfstate. A dry-run now asks the move not to run and plans against the old key while it is pending, the read-only path fluid diff already takes; the first real apply moves it. The move's probe no longer installs anything: OpenTofu's init installs the providers the state names (the latest hashicorp/aws for an aws state), and the probe ran on every apply whose new key was empty. It now inits with -plugin-dir on an empty directory and pulls from the backend that init is shown to have recorded, falling back to a plain init. The one-time copy installs what the old state names at the plugin's own pins. Using the apply's .terraform/providers as the plugin directory was tried and broke the workdir under a plugin cache, so it is not used. A remote key that does not name the provider (the shared fluid/terraform.tfstate a bucket-only --state-backend gives a contract without packaging) is refused when it holds another cloud's resources (state_shared_with_another_provider). Sovereignty. Under strict, a cloud-region binding (aws/gcp/azure) whose jurisdiction cannot be resolved is refused unless the region is named in allowedRegions; it was a warning, so me-central2, northamerica-south1 and the asia multi-region passed validate and generate iac on an EU-only contract. The nine GCP regions the vendored table lacks, and the dual-regions within one jurisdiction (EUR4, NAM4, ASIA1), are mapped from Google's own location lists, in source, not in the ODbL csv. A gcp pubsub_topic binding's region becomes message_storage_policy.allowed_persistence_regions (hashicorp/google), which the GCP hook checks; it was dropped. Command Center. A run refused before the engine registered it now carries its product: the engine identifies it as soon as the contract and provider are known, and a run refused earlier is described from the base contract. Overlays. Only the workspace's expected-environments refuses --env X with no overlay; the contract's own environments block warns again, as on 0.16.5, since forge-cli applies nothing from it. Tests: the raced and unverified branches of the move under fault injection, the probe installing nothing (real tofu, no registry reachable), a dry-run leaving the bucket's key set unchanged and the drift pass reading the old key (real tofu against moto), and each of the fixes above. --- fluid_build/cli/_apply_cc_report.py | 59 ++ fluid_build/cli/_apply_opentofu_engine.py | 98 +++- fluid_build/iac/providers/gcp.py | 15 +- fluid_build/iac/runner.py | 6 + fluid_build/iac/state_migration.py | 146 ++++- fluid_build/loader.py | 37 +- fluid_build/policy/sovereignty.py | 114 +++- fluid_build/providers/gcp/util/sovereignty.py | 33 +- .../test_apply_reports_to_command_center.py | 98 ++++ .../test_one_contract_two_clouds_pipeline.py | 42 +- .../iac/test_iac_state_key_migration_moto.py | 79 ++- tests/iac/test_state_migration_safety.py | 520 ++++++++++++++++++ .../test_sovereignty_enforcement_mode.py | 51 +- ...test_gcp_locations_and_pubsub_placement.py | 280 ++++++++++ 14 files changed, 1493 insertions(+), 85 deletions(-) create mode 100644 tests/iac/test_state_migration_safety.py create mode 100644 tests/providers/test_gcp_locations_and_pubsub_placement.py diff --git a/fluid_build/cli/_apply_cc_report.py b/fluid_build/cli/_apply_cc_report.py index be4a36ec..ff60f1d0 100644 --- a/fluid_build/cli/_apply_cc_report.py +++ b/fluid_build/cli/_apply_cc_report.py @@ -111,6 +111,29 @@ def __init__(self, args: Any, logger: logging.Logger) -> None: # -- filled by the apply engine ------------------------------------ + def identify( + self, + *, + contract: Mapping[str, Any], + provider: Optional[str] = None, + environment: Optional[str] = None, + ) -> None: + """What the run is for, as soon as the engine knows it; registers nothing. + + A run refused before :meth:`begin` (a sovereignty refusal in the + emitter, a provider that cannot be resolved) is then still reported + with its product, contract version and platform. + """ + try: + if environment: + self.environment = environment + self.metadata.update(_contract_facts(contract)) + if provider: + self.provider = provider + self.metadata["platform"] = provider + except Exception as exc: # noqa: BLE001 - reporting never fails an apply + _LOG.debug("command center report: identify failed: %s", type(exc).__name__) + def begin( self, *, @@ -163,6 +186,11 @@ def finish(self, status: str, *, event: Optional[str] = None, exit_code: int = 0 self._finished = True try: if not self._began: + if not self.metadata.get("product_id"): + # Refused before the engine read the contract (plan + # binding, a declared env with no overlay): the base + # document still says which product this run was for. + self.metadata.update(_base_contract_facts(self.contract_path)) self._register() reporter = self._reporter if reporter is None: @@ -278,6 +306,37 @@ def _contract_facts(contract: Mapping[str, Any]) -> Dict[str, Any]: return facts +def _base_contract_facts(contract_path: Optional[str]) -> Dict[str, Any]: + """Identity facts from the base contract (or a plan's embedded one), no overlay. + + For a run that failed before the engine loaded the contract, possibly + because its overlay could not be applied. The product id and versions + are the base's (an overlay rebinds, it does not rename). No contract + hash: the Command Center keys versions by the hash of the compiled + contract, which this run never produced. + """ + if not contract_path: + return {} + try: + if contract_path.endswith(".json"): + import json + + with open(contract_path, encoding="utf-8") as handle: + plan = json.load(handle) + contract = plan.get("contract") if isinstance(plan, dict) else None + else: + from fluid_build.loader import load_contract + + contract = load_contract(contract_path) + except Exception: # noqa: BLE001 - an unreadable contract is simply not described + return {} + if not isinstance(contract, Mapping): + return {} + facts = _contract_facts(contract) + facts.pop("contract_hash", None) + return {k: v for k, v in facts.items() if v is not None} + + def _command_center_config(logger: logging.Logger): """``(CommandCenterConfig, None)`` to report with, or ``(None, reason)``.""" from fluid_build.observability.config import CommandCenterConfig diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index 48834d05..1f6ce08f 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -49,6 +49,8 @@ PENDING, StateMigrationError, StateReconciliation, + other_clouds, + read_state, records_backend, ) from fluid_build.iac.state_migration import reconcile_state_key as _reconcile_state @@ -72,6 +74,14 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: contract = _load_contract(args, logger) provider = _resolve_provider(contract, getattr(args, "provider", None) or "auto") + report = current_report() + if report is not None: + # Known from here on, so a refusal before the run is registered (a + # sovereignty refusal in the emitter, an init that fails) still + # reaches the Command Center with its product, version and platform. + # A run refused earlier is described from the base contract instead + # (``ApplyRunReport.finish``). + report.identify(contract=contract, provider=provider, environment=_applied_env(args)) plugin = get_iac_plugin(provider) if plugin is None: @@ -141,7 +151,6 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: # The Command Center hears about the run now that the product, the # provider and the environment are known (best effort; see # cli/_apply_cc_report.py). Addresses and counts only, never tofu output. - report = current_report() if report is not None: report.begin( contract=contract, @@ -159,11 +168,35 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: if not init.ok: raise CLIError(1, "opentofu_init_failed", {"error": _tail(init.stderr or init.stdout)}) - # State a previous release kept at the key without the provider moves to - # this provider's key (OpenTofu's own ``init -migrate-state``), so the - # first apply after the upgrade does not plan every resource as new - # (see ``iac.state_migration``). Before anything reads the state. - reconcile_state_key(target, provider, env, logger, migrate=True) + dry_run = bool(getattr(args, "dry_run", False)) + if dry_run: + # A dry-run is plan only and writes no state, so it never moves any + # either: while the move is pending it plans against the old key, the + # read-only path ``fluid diff`` takes (``read_target``). The copy is + # left to the first real apply, which a plan-only CI role with + # read-only state access never runs. + read = read_target(target, provider, env, logger) + if read is not target: + target = read + module, actions = emit_module(plugin, contract, target, logger) + module_path.write_text(module, encoding="utf-8") + init = runner.tofu_init(str(workdir), backend=True, env=env, reconfigure=True) + if not init.ok: + raise CLIError( + 1, "opentofu_init_failed", {"error": _tail(init.stderr or init.stdout)} + ) + else: + # State a previous release kept at the key without the provider moves + # to this provider's key (OpenTofu's own ``init -migrate-state``), so + # the first apply after the upgrade does not plan every resource as + # new (see ``iac.state_migration``). Before anything reads the state. + reconcile_state_key(target, provider, env, logger, migrate=True) + + # One state, two clouds: a key that does not name the provider (the + # shared ``fluid/terraform.tfstate`` a bucket-only --state-backend gives a + # contract without packaging, or an explicit key used for both) can hold + # the other cloud's resources, which this plan would destroy. + guard_state_shared_with_another_cloud(target, provider, env) # Pre-plan region guard. A module now pins the region its bindings name, # and moving a contract's resources to it would not show as a destroy. @@ -241,7 +274,7 @@ def apply_via_opentofu(args, logger: logging.Logger) -> int: **changes, ) - if bool(getattr(args, "dry_run", False)): + if dry_run: cprint("\ndry-run: plan only — not applying.") info(logger, "opentofu_apply_dry_run", provider=provider, **changes) return 0 @@ -446,6 +479,57 @@ def reconcile_state_key( return outcome +def _key_names_provider(backend: Mapping[str, Any], provider: str) -> bool: + """True when the backend's key (or GCS prefix) has ``provider`` as a path segment.""" + if "s3" in backend: + path = str((backend.get("s3") or {}).get("key") or "") + elif "gcs" in backend: + path = str((backend.get("gcs") or {}).get("prefix") or "") + else: + return False + return provider in path.split("/") + + +def guard_state_shared_with_another_cloud( + target: StateTarget, provider: str, env: Mapping[str, str] +) -> None: + """Refuse a remote state whose key names no provider and holds another cloud's resources. + + The per-provider default keys (``fluid///...``) cannot be + shared by two clouds, and local state lives in a per-provider workdir, so + only a remote key that does not name the provider is read (one ``tofu + state pull``). Whose resources they are is the migration's own rule + (``state_migration.other_clouds``). Without this, the gcp plan on a key + the aws apply wrote reads the aws resources as orphans, and + ``--allow-data-loss`` destroys them. + """ + if target.backend is None or _key_names_provider(target.backend, provider): + return + location = backend_location(target.backend) + try: + doc = read_state(target.workdir, env) + except StateMigrationError as exc: + raise CLIError(1, exc.code, {"error": str(exc), "state": location}) + others = sorted(other_clouds(doc.resources, provider)) + if not others: + return + raise CLIError( + 1, + "state_shared_with_another_provider", + { + "error": ( + f"{location} holds resources of the {', '.join(others)} provider, and this is " + f"the {provider} apply: its plan would read them as orphans to destroy. Give " + "each provider its own state, with a key that names the provider in " + "--state-backend (for example fluid///terraform.tfstate) or a " + f"bucket-only {STATE_BACKEND_ENV}, which keys state by contract and provider" + ), + "state": location, + "providers": others, + }, + ) + + def read_target( target: StateTarget, provider: str, env: Mapping[str, str], logger: logging.Logger ) -> StateTarget: diff --git a/fluid_build/iac/providers/gcp.py b/fluid_build/iac/providers/gcp.py index 1c156735..0bc291d2 100644 --- a/fluid_build/iac/providers/gcp.py +++ b/fluid_build/iac/providers/gcp.py @@ -1023,10 +1023,17 @@ def _emit_pubsub( ) -> None: topic = loc.get("topic") or f"{cid}-topic" topic_res = safe_ident(f"{cid}_{topic}") - resources.setdefault("google_pubsub_topic", {})[topic_res] = { - "name": topic, - "labels": labels, - } + body: Dict[str, Any] = {"name": topic, "labels": labels} + # The binding's region is where the topic's messages may be stored: + # hashicorp/google ``google_pubsub_topic.message_storage_policy`` + # (``allowed_persistence_regions``). It was dropped, so a topic bound to + # europe-west1 under an EU-only sovereignty block passed validate and + # stored messages wherever Pub/Sub chose. The GCP sovereignty hook reads + # the same field back (``resource_placements``). + region = loc.get("region") or loc.get("location") + if region: + body["message_storage_policy"] = {"allowed_persistence_regions": [str(region)]} + resources.setdefault("google_pubsub_topic", {})[topic_res] = body subscription = loc.get("subscription") if subscription: resources.setdefault("google_pubsub_subscription", {})[ diff --git a/fluid_build/iac/runner.py b/fluid_build/iac/runner.py index b24ff078..56c15e13 100644 --- a/fluid_build/iac/runner.py +++ b/fluid_build/iac/runner.py @@ -206,6 +206,7 @@ def tofu_init( env: Optional[Mapping[str, str]] = None, reconfigure: bool = False, force_copy: bool = False, + plugin_dir: Optional[str] = None, ): """``tofu init`` — install providers (and initialise the backend). @@ -215,6 +216,9 @@ def tofu_init( copy the recorded backend's state into the module's backend without a prompt, overwriting whatever the destination holds (OpenTofu ``backendMigrateState_s_s``), so a caller checks the destination first. + ``plugin_dir`` passes ``-plugin-dir``: providers come only from that + directory, "as if it had been configured as a ``filesystem_mirror``" + (opentofu.org/docs/cli/commands/init), so nothing is downloaded. """ args = ["init", "-input=false", "-no-color"] if not backend: @@ -223,6 +227,8 @@ def tofu_init( args.append("-reconfigure") if force_copy: args.append("-force-copy") + if plugin_dir: + args.append(f"-plugin-dir={plugin_dir}") return _run(args, workdir=workdir, env=env, command="init") diff --git a/fluid_build/iac/state_migration.py b/fluid_build/iac/state_migration.py index d077d8a0..f0d58a8e 100644 --- a/fluid_build/iac/state_migration.py +++ b/fluid_build/iac/state_migration.py @@ -31,8 +31,26 @@ ``backendMigrateState_s_s``). The copy is checked afterwards by reading it back: the same resources, since OpenTofu gives a copy into an empty destination a fresh lineage. Terragrunt's ``backend migrate`` wraps the same -step for a renamed unit; this is that idea without the wrapper. The scratch -directory holds no provider, so the step downloads nothing. +step for a renamed unit; this is that idea without the wrapper. + +**What the scratch ``tofu init`` installs.** OpenTofu's init installs every +provider the *state* names, not only the module's: a ``{"terraform": {}}`` +module beside a state naming ``hashicorp/null`` installed the latest +``hashicorp/null`` (measured, tofu 1.12), and for an aws state that is the +latest ``hashicorp/aws``, not the pinned ``~> 5.0``. The probe runs on every +apply whose new key is still empty (every ``--dry-run`` while a move is +pending, every gcp run while the old key holds the aws state), so it installs +nothing: ``-plugin-dir`` names an empty directory, the init stops at its +provider step after its backend step recorded the old backend, and the state +is pulled from exactly that recorded backend (checked, never assumed). +Attribution needs the document, not the providers. Should a future OpenTofu +not record the backend first, the probe falls back to a plain init, which +may install, rather than fail. The apply's own ``.terraform/providers`` is +never the plugin directory: with a plugin cache configured its entries link +into the cache, and an init reading them as a mirror broke the workdir +(measured: "no package for hashicorp/aws 5.100.0 cached"). The copy itself, +once per contract, installs what the old state names at the plugin's own +pins (``required_providers`` of the IaC plugin), never the latest. **Never lose it, never guess.** ``-force-copy`` overwrites a destination that holds state (the same OpenTofu function skips its confirmation), so the copy @@ -63,7 +81,7 @@ import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, FrozenSet, Iterable, Mapping, Optional, Tuple +from typing import Any, Dict, FrozenSet, Iterable, Mapping, Optional, Set, Tuple from . import runner from .backend import backend_location @@ -179,20 +197,13 @@ def classify(resources: Iterable[Mapping[str, Any]], provider: str) -> Tuple[str ``detail`` names the owner for ``"other"`` and the reason for ``"ambiguous"``. See the module docstring for the rule. """ - from .registry import IAC_PLUGINS - sources = state_sources(resources) - by_plugin = {name: plugin_sources(name) for name in IAC_PLUGINS} - by_plugin.setdefault(provider, plugin_sources(provider)) + by_plugin = _sources_by_plugin(provider) known = frozenset().union(*by_plugin.values()) unknown = sources - known if unknown: return "ambiguous", "providers no forge-cli IaC plugin emits: " + ", ".join(sorted(unknown)) - owners = set() - for name, mine in by_plugin.items(): - exclusive = mine - frozenset().union(*(s for n, s in by_plugin.items() if n != name)) - if sources & exclusive: - owners.add(name) + owners = _owners(sources, by_plugin) if owners == {provider} and sources <= by_plugin[provider]: return "mine", provider if len(owners) == 1 and provider not in owners: @@ -204,6 +215,35 @@ def classify(resources: Iterable[Mapping[str, Any]], provider: str) -> Tuple[str return "ambiguous", "only providers no single cloud owns (" + ", ".join(sorted(sources)) + ")" +def other_clouds(resources: Iterable[Mapping[str, Any]], provider: str) -> FrozenSet[str]: + """The IaC plugins other than ``provider`` whose own resources the state holds. + + A resource counts for a plugin when its provider source is one no other + plugin emits (``hashicorp/google`` is gcp's; ``hashicorp/null`` is no + one's), the rule :func:`classify` attributes a state by. + """ + by_plugin = _sources_by_plugin(provider) + return frozenset(_owners(state_sources(resources), by_plugin) - {provider}) + + +def _sources_by_plugin(provider: str) -> Dict[str, FrozenSet[str]]: + from .registry import IAC_PLUGINS + + by_plugin = {name: plugin_sources(name) for name in IAC_PLUGINS} + by_plugin.setdefault(provider, plugin_sources(provider)) + return by_plugin + + +def _owners(sources: FrozenSet[str], by_plugin: Mapping[str, FrozenSet[str]]) -> Set[str]: + """Plugins owning a source in ``sources`` that no other plugin emits.""" + owners: Set[str] = set() + for name, mine in by_plugin.items(): + exclusive = mine - frozenset().union(*(s for n, s in by_plugin.items() if n != name)) + if sources & exclusive: + owners.add(name) + return owners + + def reconcile_state_key( *, workdir: Path, @@ -220,7 +260,8 @@ def reconcile_state_key( ``current``: the new key is read there, so an apply whose state is already at the new key pays one ``tofu state pull`` and nothing else. Only a new key with no state brings the scratch directory (and its - ``tofu init``, which installs the providers the old state names) in. + ``tofu init -plugin-dir``, which installs nothing) in; only a move + installs, at the plugin's pins. ``current`` and ``legacy`` are ``terraform.backend`` blocks (:func:`.backend.parse_backend`). Returns what was found; raises :class:`StateMigrationError` when the old state cannot be attributed or @@ -233,8 +274,7 @@ def reconcile_state_key( if now.exists: return StateReconciliation(CURRENT, legacy, current) with tempfile.TemporaryDirectory(prefix="fluid-state-") as tmp: - legacy_dir = Path(tmp) / "legacy" - old = _probe(legacy_dir, legacy, env) + old = _probe(Path(tmp), legacy, env) if not old.exists or not old.resources: return StateReconciliation(NOTHING, legacy, current) verdict, detail = classify(old.resources, provider) @@ -263,8 +303,21 @@ def reconcile_state_key( f"{backend_location(current)} received a different state while this apply " f"was about to move {backend_location(legacy)} there; nothing was moved", ) - _write_backend(legacy_dir, current) - init = runner.tofu_init(str(legacy_dir), env=env, force_copy=True) + # The copy starts from a directory initialised on the old key, whose + # module pins the providers the old state names to the plugin's own + # versions (see the module docstring). + move_dir = Path(tmp) / "move" + pins = plugin_pins(provider, state_sources(old.resources)) + _write_backend(move_dir, legacy, pins) + first = runner.tofu_init(str(move_dir), env=env) + if not first.ok: + raise StateMigrationError( + "state_migration_failed", + f"could not initialise on {backend_location(legacy)} to move it: " + + _tail(first.stderr or first.stdout), + ) + _write_backend(move_dir, current, pins) + init = runner.tofu_init(str(move_dir), env=env, force_copy=True) if not init.ok: raise StateMigrationError( "state_migration_failed", @@ -288,22 +341,67 @@ def reconcile_state_key( return result -def _write_backend(workdir: Path, backend: Mapping[str, Any]) -> None: +def _write_backend( + workdir: Path, + backend: Mapping[str, Any], + required_providers: Optional[Mapping[str, Any]] = None, +) -> None: workdir.mkdir(parents=True, exist_ok=True) - doc = {"terraform": {"backend": dict(backend)}} + terraform: Dict[str, Any] = {"backend": dict(backend)} + if required_providers: + terraform["required_providers"] = dict(required_providers) + doc = {"terraform": terraform} (workdir / "main.tf.json").write_text(json.dumps(doc, indent=2), encoding="utf-8") -def _probe(workdir: Path, backend: Mapping[str, Any], env: Mapping[str, str]) -> StateDoc: - """Initialise a provider-less module on ``backend`` and pull its state.""" - _write_backend(workdir, backend) - init = runner.tofu_init(str(workdir), env=env) +def plugin_pins(provider: str, sources: Iterable[str]) -> Dict[str, Dict[str, str]]: + """``provider``'s ``required_providers`` entries for the given ``namespace/type`` sources.""" + from .registry import get_iac_plugin + + wanted = {str(s).lower() for s in sources} + required = getattr(get_iac_plugin(provider), "required_providers", None) or {} + return { + name: dict(spec) + for name, spec in required.items() + if str(spec.get("source", "")).lower() in wanted + } + + +def _probe(tmp: Path, backend: Mapping[str, Any], env: Mapping[str, str]) -> StateDoc: + """``backend``'s state, read with nothing installed (see the module docstring). + + ``tofu init -plugin-dir`` with an empty directory: a state that names a + provider stops the init after the backend step, and the state is read + only when the init is shown to have recorded ``backend``. Otherwise a + plain init in a fresh directory is tried, as before; its failure is the + error. + """ + empty = tmp / "no-providers" + empty.mkdir(parents=True, exist_ok=True) + probe_dir = tmp / "legacy" + _write_backend(probe_dir, backend) + init = runner.tofu_init(str(probe_dir), env=env, plugin_dir=str(empty)) + if init.ok or records_backend(probe_dir, backend): + return _pull(probe_dir, env) + plain_dir = tmp / "legacy-plain" + _write_backend(plain_dir, backend) + init = runner.tofu_init(str(plain_dir), env=env) if not init.ok: raise StateMigrationError( "state_migration_probe_failed", f"could not read {backend_location(backend)}: " + _tail(init.stderr or init.stdout), ) - return _pull(workdir, env) + return _pull(plain_dir, env) + + +def read_state(workdir: Path, env: Mapping[str, str]) -> StateDoc: + """The state ``workdir``'s initialised backend holds (``tofu state pull``). + + Raises :class:`StateMigrationError` (``state_migration_probe_failed``) + when it cannot be read; the error carries stderr only, never the + document. + """ + return _pull(Path(workdir), env) def _pull(workdir: Path, env: Mapping[str, str]) -> StateDoc: diff --git a/fluid_build/loader.py b/fluid_build/loader.py index 7e07d0f9..2417e3fc 100644 --- a/fluid_build/loader.py +++ b/fluid_build/loader.py @@ -500,6 +500,10 @@ def _base_platforms(contract: Mapping[str, Any]) -> List[str]: return out +#: How :func:`declared_environments` names the contract's own block. +CONTRACT_ENVIRONMENTS_BLOCK = "the contract's environments block" + + def declared_environments( contract_path: str | Path, contract: Mapping[str, Any] ) -> List[Tuple[str, List[str]]]: @@ -514,7 +518,7 @@ def declared_environments( found: List[Tuple[str, List[str]]] = [] environments = contract.get("environments") if isinstance(environments, dict) and environments: - found.append(("the contract's environments block", [str(k) for k in environments])) + found.append((CONTRACT_ENVIRONMENTS_BLOCK, [str(k) for k in environments])) try: from .util.workspace_root import WORKSPACE_CONFIG_FILENAME, find_workspace_root @@ -546,13 +550,19 @@ def declared_environments( def refuse_declared_missing_overlay( contract_path: str | Path, env: str, contract: Mapping[str, Any] ) -> None: - """Refuse ``--env `` with no overlay when the product declares ``env``. + """Refuse ``--env `` with no overlay when the workspace expects ``env``. Without an overlay the base contract is used unchanged, which for a declared environment means deploying it as if it were that environment (measured: silver ``--env gcp`` validated and planned the local base, - rc=0). The base-by-convention ``dev`` and an env the base contract is - already bound to (``local`` for a local base) are the base, and pass. + rc=0). The declaration that refuses is the workspace's + ``expected-environments``: a statement that this product has one overlay + per environment. The contract's own ``environments`` block does not + refuse. It is schema-valid, forge-cli applies nothing from it, and + refusing on it broke contracts that validated before, so + :func:`note_missing_overlay` names it in its warning instead. The + base-by-convention ``dev`` and an env the base contract is already bound + to (``local`` for a local base) are the base, and pass. """ if env == BASE_ENV_BY_CONVENTION: return @@ -560,6 +570,8 @@ def refuse_declared_missing_overlay( if env in platforms: return for source, envs in declared_environments(contract_path, contract): + if source == CONTRACT_ENVIRONMENTS_BLOCK: + continue if env in envs: from ._contract_loader import CLIError @@ -602,8 +614,10 @@ def note_missing_overlay( once per (contract, env) per process — one command loads the same contract several times. - ``contract`` (the base) enables the refusal: an env the product declares - (:func:`refuse_declared_missing_overlay`) is an error, every time. + ``contract`` (the base) enables the refusal: an env the workspace + expects (:func:`refuse_declared_missing_overlay`) is an error, every + time. An env only the contract's ``environments`` block names is said in + the warning. """ log = logger or LOG if contract is not None: @@ -625,19 +639,28 @@ def note_missing_overlay( return existing = available_overlay_envs(contract_key) platforms = _base_platforms(contract) if contract is not None else [] + environments = contract.get("environments") if contract is not None else None + in_block = isinstance(environments, dict) and env in environments log.warning( "overlay_not_found: --env %r matched no overlay for %s, so the BASE contract is " - "used unchanged%s. Overlays that exist: %s. Add an overlay for it under overlays/ " + "used unchanged%s.%s Overlays that exist: %s. Add an overlay for it under overlays/ " "or pass one of the existing environments.", env, contract_key, f" (it binds to {', '.join(platforms)}, not to {env!r})" if platforms else "", + ( + f" The contract's environments block names {env!r}, but forge-cli applies " + "nothing from that block; an overlay is what changes a binding." + if in_block + else "" + ), ", ".join(existing) if existing else "none", extra={ "event": "overlay_not_found", "env": env, "available_envs": existing, "base_platforms": platforms, + "declared_in_environments_block": in_block, }, ) diff --git a/fluid_build/policy/sovereignty.py b/fluid_build/policy/sovereignty.py index 3efbb8c7..f5a453bc 100644 --- a/fluid_build/policy/sovereignty.py +++ b/fluid_build/policy/sovereignty.py @@ -27,7 +27,7 @@ from enum import Enum from pathlib import Path from types import MappingProxyType -from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import AbstractSet, Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Set, Tuple from ._common import iter_exposes @@ -182,6 +182,7 @@ def region_jurisdiction_map() -> Mapping[str, str]: for provider in ("aws", "gcp", "azure"): table.update(_load_vendored(provider)) table.update(SovereigntyValidator.VENDORED_CORRECTIONS) + table.update(_gcp_locations_not_vendored()) table.update(_MULTI_REGION_JURISDICTIONS) table.update(_load_botocore_aws()) # Read-only: the cached object is shared process-wide (and re-exported by @@ -375,10 +376,18 @@ def validate(self, contract: Dict[str, Any]) -> Tuple[bool, List[SovereigntyViol sovereignty = contract.get("sovereignty") if not sovereignty: return True, [] # No sovereignty constraints = always valid - return self.check_placements(sovereignty, contract_placements(contract)) + return self.check_placements( + sovereignty, + contract_placements(contract), + region_placed=region_placed_exposes(contract), + ) def check_placements( - self, sovereignty: Mapping[str, Any], placements: Sequence[Tuple[str, Optional[str]]] + self, + sovereignty: Mapping[str, Any], + placements: Sequence[Tuple[str, Optional[str]]], + *, + region_placed: AbstractSet[str] = frozenset(), ) -> Tuple[bool, List[SovereigntyViolation]]: """Evaluate ``sovereignty`` against each ``(where, region)`` placement. @@ -388,6 +397,11 @@ def check_placements( ``location``), so a region the provider filled in by default is checked where it is used. A placement whose region is ``None`` is a binding on a region-placed platform that names none. + + ``region_placed`` names the placements (their ``where``) that sit on a + :data:`REGION_PLACED_PLATFORMS` cloud. For those, a strict + jurisdiction refuses a region whose jurisdiction cannot be resolved + (check 3); elsewhere that stays a warning. """ violations = [] @@ -482,25 +496,42 @@ def check_placements( if jurisdiction and jurisdiction not in UNCONSTRAINED_JURISDICTIONS: region_jurisdiction = region_jurisdiction_map().get(region, "Unknown") if region_jurisdiction != jurisdiction and region_jurisdiction != "Global": - # "Unknown" is an inability to evaluate, not a violation, and - # the two must not be conflated: a region the vendored table - # does not carry says nothing about where it actually is. - # Escalating it under strict would fail closed on every - # contract using an unmapped region — defensible for a - # sovereignty control, but a separate decision with its own - # blast radius, not a side effect of making enforcementMode - # mean what the schema says. Check 4 already draws this exact - # line and refuses to let one Unknown agree with another. + # "Unknown" is an inability to evaluate, not a violation: a + # region the table does not carry says nothing about where + # it is. On a cloud region (aws / gcp / azure) under + # strict, though, a jurisdiction the policy cannot show is + # a place it cannot allow: the GCP table lagged Google by + # nine regions and the ASIA multi-region, and each went + # through validate and `generate iac` with a warning + # (me-central2 on an EU-only contract, measured). So + # strict refuses it there, unless the operator vouched for + # the region by naming it in allowedRegions. Advisory and + # audit, and every other platform, keep the warning. + # Check 4 still refuses to let one Unknown agree with + # another. unresolvable = region_jurisdiction == "Unknown" + fail_closed = ( + unresolvable + and enforcement_mode is EnforcementMode.STRICT + and expose_id in region_placed + and region not in allowed_regions + ) violations.append( SovereigntyViolation( severity=( - "warning" if unresolvable else severity_for(enforcement_mode) + "warning" + if unresolvable and not fail_closed + else severity_for(enforcement_mode) ), message=f"Region '{region}' (jurisdiction: {region_jurisdiction}) " f"does not match required jurisdiction: {jurisdiction}", expose_id=expose_id, - suggestion=f"Consider using regions in {jurisdiction} jurisdiction", + suggestion=( + f"Use a region in the {jurisdiction} jurisdiction; if " + f"'{region}' is one, name it in sovereignty.allowedRegions" + if fail_closed + else f"Consider using regions in {jurisdiction} jurisdiction" + ), ) ) @@ -598,6 +629,61 @@ def check_placements( _MULTI_REGION_JURISDICTIONS = {"US": "US", "EU": "EU", "us": "US", "eu": "EU"} +#: GCP locations the vendored dataset (dgl/cloud-regions) does not carry, by +#: the countries their data centres are in, from Google's own location lists +#: (docs.cloud.google.com/bigquery/docs/locations and +#: /storage/docs/locations, read 2026-09-28). Kept here, not patched into the +#: ODbL csv, for the reason ``VENDORED_CORRECTIONS`` gives. A location +#: resolves only when every country it spans is in one jurisdiction: the +#: dual-regions EUR5 (Belgium + London), EUR7 (London + Frankfurt) and EUR8 +#: (Frankfurt + Zürich) span two and stay Unknown, as does the ASIA +#: multi-region ("data centres in Asia", several countries), which a strict +#: jurisdiction then refuses on a cloud region (check 3). +_GCP_LOCATION_COUNTRIES: Dict[str, Tuple[str, ...]] = { + "africa-south1": ("za",), # Johannesburg + "asia-southeast3": ("th",), # Bangkok + "europe-north2": ("se",), # Stockholm + "europe-west10": ("de",), # Berlin + "europe-west12": ("it",), # Turin + "me-central1": ("qa",), # Doha + "me-central2": ("sa",), # Dammam + "me-west1": ("il",), # Tel Aviv + "northamerica-south1": ("mx",), # Mexico + # Cloud Storage predefined dual-regions (either case is accepted). + "asia1": ("jp",), # Tokyo + Osaka + "eur4": ("fi", "nl"), # Finland + Netherlands + "eur5": ("be", "uk"), # Belgium + London + "eur7": ("uk", "de"), # London + Frankfurt + "eur8": ("de", "ch"), # Frankfurt + Zürich + "nam4": ("us",), # Iowa + South Carolina +} + + +def _gcp_locations_not_vendored() -> Dict[str, str]: + """``location -> jurisdiction`` for :data:`_GCP_LOCATION_COUNTRIES`.""" + resolved: Dict[str, str] = {} + for location, countries in _GCP_LOCATION_COUNTRIES.items(): + found = {SovereigntyValidator.COUNTRY_JURISDICTIONS.get(c) for c in countries} + jurisdiction = found.pop() if len(found) == 1 else None + if jurisdiction is None: + continue + resolved[location] = jurisdiction + if location.isalnum(): # a dual-region code: EUR4 and eur4 alike + resolved[location.upper()] = jurisdiction + return resolved + + +def region_placed_exposes(contract: Mapping[str, Any]) -> FrozenSet[str]: + """``exposeId`` of every expose bound to a :data:`REGION_PLACED_PLATFORMS` cloud.""" + out: Set[str] = set() + for expose in iter_exposes(dict(contract)): + binding = expose.get("binding") or {} + if isinstance(binding, Mapping): + if str(binding.get("platform") or "").lower() in REGION_PLACED_PLATFORMS: + out.add(str(expose.get("exposeId", "unknown"))) + return frozenset(out) + + def binding_region(binding: Mapping[str, Any]) -> Optional[str]: """The region a binding places its data in, read where its emitter reads it. diff --git a/fluid_build/providers/gcp/util/sovereignty.py b/fluid_build/providers/gcp/util/sovereignty.py index 018bfebf..415d55dc 100644 --- a/fluid_build/providers/gcp/util/sovereignty.py +++ b/fluid_build/providers/gcp/util/sovereignty.py @@ -78,7 +78,10 @@ def resource_placements(resources: Mapping[str, Any]) -> List[Placement]: """``(address, location)`` for every emitted resource that names one. ``resources`` is the plugin's ``{type: {name: body}}``. A value that is an - OpenTofu reference (``${...}``) is not a place and is skipped. + OpenTofu reference (``${...}``) is not a place and is skipped. A Pub/Sub + topic has no ``location``: where its messages are stored is its + ``message_storage_policy.allowed_persistence_regions``, one placement + per region. """ out: List[Placement] = [] for rtype, by_name in (resources or {}).items(): @@ -89,9 +92,27 @@ def resource_placements(resources: Mapping[str, Any]) -> List[Placement]: continue for key in ("location", "region"): value = body.get(key) - if isinstance(value, str) and value and not value.startswith("${"): - out.append((f"{rtype}.{name}", value)) + if _is_place(value): + out.append((f"{rtype}.{name}", str(value))) break + for region in _persistence_regions(body): + out.append((f"{rtype}.{name}", region)) + return out + + +def _is_place(value: Any) -> bool: + return isinstance(value, str) and bool(value) and not value.startswith("${") + + +def _persistence_regions(body: Mapping[str, Any]) -> List[str]: + """``message_storage_policy.allowed_persistence_regions`` (block as object or list).""" + policy = body.get("message_storage_policy") + blocks = policy if isinstance(policy, list) else [policy] + out: List[str] = [] + for block in blocks: + if isinstance(block, Mapping): + regions = block.get("allowed_persistence_regions") or [] + out.extend(str(r) for r in regions if _is_place(r)) return out @@ -120,7 +141,11 @@ def gcp_sovereignty_violations( everything = unplaced_gcp_exposes(contract) + list(placements) if not everything: return [] - _, violations = SovereigntyValidator().check_placements(sovereignty, everything) + # Every placement here is a GCP one, so a jurisdiction the table cannot + # resolve is refused under strict, as for any cloud region (check 3). + _, violations = SovereigntyValidator().check_placements( + sovereignty, everything, region_placed={where for where, _ in everything} + ) return violations diff --git a/tests/cli/test_apply_reports_to_command_center.py b/tests/cli/test_apply_reports_to_command_center.py index 36aa96e1..633d3805 100644 --- a/tests/cli/test_apply_reports_to_command_center.py +++ b/tests/cli/test_apply_reports_to_command_center.py @@ -374,3 +374,101 @@ def test_a_private_address_is_refused_before_the_credential_is_sent( assert _apply(product) == 0 assert called == [] assert "private or metadata address" in " ".join(capsys.readouterr().out.split()) + + +# ── A run refused before the engine registered it still names its product ── + + +def test_a_sovereignty_refusal_is_reported_with_its_product(product, tofu, cc, monkeypatch): + """The gcp overlay loses its region under a strict policy: the emitter + refuses before the run is registered, and the run still says whose it is.""" + from fluid_build._errors import SovereigntyViolationError + + url, recorder = cc + _configure(monkeypatch, url) + doc = yaml.safe_load(product.read_text(encoding="utf-8")) + doc["sovereignty"] = { + "jurisdiction": "EU", + "allowedRegions": ["europe-west1"], + "enforcementMode": "strict", + } + product.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + overlay = product.parent / "overlays" / "gcp.yaml" + placed = yaml.safe_load(overlay.read_text(encoding="utf-8")) + placed["exposes"][0]["binding"]["location"].pop("region") + overlay.write_text(yaml.safe_dump(placed), encoding="utf-8") + + with pytest.raises(SovereigntyViolationError): + _apply(product) + + (post,) = _by(recorder, "POST") + (patch,) = _by(recorder, "PATCH") + assert post["body"]["provider"] == "gcp" + assert post["body"]["environment"] == "gcp" + metadata = post["body"]["metadata"] + assert metadata["product_id"] == "bronze.customer_subscriptions" + assert metadata["contract_version"] == "1.4.0" + assert metadata["platform"] == "gcp" + assert metadata["contract_hash"] + assert patch["body"]["status"] == "failed" + assert patch["body"]["result"]["error_event"] == "SovereigntyViolationError" + + +def test_a_run_refused_before_the_contract_loads_names_the_base_product( + product, tofu, cc, monkeypatch +): + """``--env prod`` that the workspace expects, with no prod overlay: the + loader refuses, and the run is still attached to the product.""" + url, recorder = cc + _configure(monkeypatch, url) + (product.parent / "fluid.workspace.yaml").write_text( + yaml.safe_dump( + { + "workspace": {"name": "cc"}, + "expected-environments": {"bronze.customer_subscriptions": ["gcp", "prod"]}, + } + ), + encoding="utf-8", + ) + from fluid_build import loader + from fluid_build.cli import build_parser + + loader._NOTED_MISSING_OVERLAYS.clear() + args = build_parser().parse_args( + ["apply", str(product), "--env", "prod", "--yes", "--no-verify-federation"] + ) + with pytest.raises(CLIError) as exc: + args.func(args, _LOG) + assert exc.value.event == "overlay_declared_but_missing" + + (post,) = _by(recorder, "POST") + (patch,) = _by(recorder, "PATCH") + metadata = post["body"]["metadata"] + assert metadata["product_id"] == "bronze.customer_subscriptions" + assert metadata["contract_version"] == "1.4.0" + assert post["body"]["environment"] == "prod" + # No platform (no overlay settled one) and no hash of a contract never compiled. + assert "contract_hash" not in metadata + assert post["body"]["provider"] is None + assert patch["body"]["result"]["error_event"] == "overlay_declared_but_missing" + + +def test_a_provider_that_cannot_be_resolved_is_reported_with_its_product( + product, tofu, cc, monkeypatch +): + """``--provider aws`` against the gcp overlay: refused while the apply + picks its engine, before the engine runs; the base names the product.""" + url, recorder = cc + _configure(monkeypatch, url) + + with pytest.raises(CLIError) as exc: + _apply(product, "--provider", "aws") + assert exc.value.event == "generate_iac_provider_mismatch" + + (post,) = _by(recorder, "POST") + metadata = post["body"]["metadata"] + assert metadata["product_id"] == "bronze.customer_subscriptions" + assert metadata["contract_version"] == "1.4.0" + assert "platform" not in metadata + assert "contract_hash" not in metadata + assert post["body"]["environment"] == "gcp" diff --git a/tests/cli/test_one_contract_two_clouds_pipeline.py b/tests/cli/test_one_contract_two_clouds_pipeline.py index 9d5e953e..b13950f5 100644 --- a/tests/cli/test_one_contract_two_clouds_pipeline.py +++ b/tests/cli/test_one_contract_two_clouds_pipeline.py @@ -21,9 +21,10 @@ * Stage 6: the generated plan stage did not run ``--check-sovereignty``, for any CI system, so a strict sovereignty violation first failed at apply. * ``--env gcp`` with no gcp overlay applied the local base unchanged with a - warning (measured: silver validate and plan rc 0). When the product - declares gcp (its ``environments`` block, or the workspace's - ``expected-environments``, which fluid-demo-env keeps) it is now an error. + warning (measured: silver validate and plan rc 0). When the workspace's + ``expected-environments`` (which fluid-demo-env keeps) declares gcp for the + product it is now an error. The contract's own ``environments`` block only + warns: forge-cli applies nothing from it. """ from __future__ import annotations @@ -253,12 +254,35 @@ def test_an_undeclared_env_still_warns_and_says_what_the_base_binds_to(silver, c assert "it binds to local, not to 'prod'" in record.getMessage() -def test_the_contract_s_own_environments_block_declares_too(silver): - from fluid_build._contract_loader import CLIError +def test_the_contract_s_own_environments_block_warns_and_does_not_refuse(silver, caplog): + """A schema-valid ``environments`` block keeps validating, as on 0.16.5. + + forge-cli applies nothing from that block, so refusing on it offered one + fix only: deleting a valid declaration. Only the workspace's + ``expected-environments`` refuses; the block is named in the warning. + """ from fluid_build.loader import load_with_overlay - doc = dict(_SILVER, environments={"staging": {}}) + doc = dict(_SILVER, environments={"staging": {}, "prod": {}}) silver.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") - with pytest.raises(CLIError) as exc: - load_with_overlay(silver, "staging") - assert exc.value.context["declared_by"] == "the contract's environments block" + caplog.set_level(logging.WARNING, logger="fluid.loader") + base = load_with_overlay(silver, "staging") + assert base["exposes"][0]["binding"]["platform"] == "local" + (record,) = [r for r in caplog.records if "overlay_not_found" in r.getMessage()] + assert "environments block names 'staging'" in record.getMessage() + assert record.declared_in_environments_block is True + + +def test_validate_env_named_only_by_the_environments_block_passes(tmp_path, monkeypatch): + """No workspace, an ``environments`` block, no overlays: rc 0, with a warning.""" + from fluid_build import loader + from fluid_build.cli import main + + loader._NOTED_MISSING_OVERLAYS.clear() + path = tmp_path / "contract.fluid.yaml" + path.write_text( + yaml.safe_dump(dict(_SILVER, environments={"staging": {}, "prod": {}}), sort_keys=False), + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + assert main(["validate", str(path), "--env", "prod"]) == 0 diff --git a/tests/iac/test_iac_state_key_migration_moto.py b/tests/iac/test_iac_state_key_migration_moto.py index 339088e8..f342b8b4 100644 --- a/tests/iac/test_iac_state_key_migration_moto.py +++ b/tests/iac/test_iac_state_key_migration_moto.py @@ -30,7 +30,10 @@ workdir (CI) as well as a kept one, the data-loss gate still closed after the move, another provider's state at the old key left alone, a state of two clouds refused, and a new key that already holds state never -overwritten. +overwritten. And the read-only paths: ``fluid apply --dry-run`` (the +generated Jenkins default) and the ``fluid diff`` / ``verify --state-drift`` +pass plan against the old key while the move is pending and write nothing; +the dry-run used to copy the state and write the new key. Skipped unless ``tofu`` is on PATH and moto's ``server`` extra is installed. """ @@ -137,6 +140,10 @@ def _object(endpoint: str, key: str) -> Dict[str, Any]: return json.loads(body) +def _addresses(state: Dict[str, Any]) -> list: + return sorted((r["mode"], r["type"], r["name"]) for r in state["resources"]) + + def _keys(endpoint: str) -> set: listing = _s3(endpoint).list_objects_v2(Bucket=_STATE_BUCKET) return {o["Key"] for o in listing.get("Contents", [])} @@ -201,9 +208,10 @@ def _apply_as_the_old_release(contract_path: Path, root: Path) -> None: _apply(contract_path, root, state_backend=f"s3://{_STATE_BUCKET}/{_LEGACY_KEY}") -def _upgraded_plan(contract_path: Path, root: Path, monkeypatch) -> None: +def _upgraded_apply(contract_path: Path, root: Path, monkeypatch, *, dry_run=False) -> None: + """The first apply of the upgraded release (a real one moves the state).""" monkeypatch.setenv("FLUID_STATE_BACKEND", f"s3://{_STATE_BUCKET}") - _apply(contract_path, root, dry_run=True) + _apply(contract_path, root, dry_run=dry_run) def _block(key: str) -> Dict[str, Any]: @@ -244,7 +252,7 @@ def test_the_old_state_moves_and_the_plan_after_it_changes_nothing( shutil.rmtree(tmp_path / ".fluid") capsys.readouterr() - _upgraded_plan(contract, tmp_path, monkeypatch) + _upgraded_apply(contract, tmp_path, monkeypatch) # The console wraps long lines; the checks read the text unwrapped. printed = capsys.readouterr().out.replace("\n", "") @@ -252,18 +260,73 @@ def test_the_old_state_moves_and_the_plan_after_it_changes_nothing( assert f"state move: moved {len(old['resources'])} resource(s)" in printed assert "tofu plan: +0 ~0 -0" in printed moved = _object(moto, _AWS_KEY) - # OpenTofu writes the copy under a fresh lineage; the resources are the same. - assert moved["resources"] == old["resources"] + # The same resources (the move verified them exactly before the apply, + # whose refresh then rewrote the object at the new key). + assert _addresses(moved) == _addresses(old) # Never lost: the old object is left exactly where it was. assert _object(moto, _LEGACY_KEY) == old - # The next apply finds its state at the new key and moves nothing. - _upgraded_plan(contract, tmp_path, monkeypatch) + # The next run finds its state at the new key and moves nothing. + _upgraded_apply(contract, tmp_path, monkeypatch, dry_run=True) again = capsys.readouterr().out.replace("\n", "") assert "state move:" not in again assert "tofu plan: +0 ~0 -0" in again +def test_a_dry_run_moves_nothing_and_plans_on_the_old_key(moto, tmp_path, monkeypatch, capsys): + """``fluid apply --dry-run`` is plan only: while the move is pending it + reads the old key, writes no object, and leaves the move to the first + real apply. It used to copy the state (measured: the new key appeared).""" + contract = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract, tmp_path) + old = _object(moto, _LEGACY_KEY) + before = _keys(moto) + capsys.readouterr() + + _upgraded_apply(contract, tmp_path, monkeypatch, dry_run=True) + + printed = capsys.readouterr().out.replace("\n", "") + assert _keys(moto) == before, f"a dry-run wrote {sorted(_keys(moto) - before)}" + assert _object(moto, _LEGACY_KEY) == old + assert f"read from s3://{_STATE_BUCKET}/{_LEGACY_KEY}" in printed + assert "tofu plan: +0 ~0 -0" in printed + assert "dry-run: plan only" in printed + + # The first real apply, from the same (kept) workdir, does the move. + _upgraded_apply(contract, tmp_path, monkeypatch) + after = capsys.readouterr().out.replace("\n", "") + assert f"state move: moved {len(old['resources'])} resource(s)" in after + assert "tofu plan: +0 ~0 -0" in after + assert _addresses(_object(moto, _AWS_KEY)) == _addresses(old) + assert _object(moto, _LEGACY_KEY) == old + + +def test_the_drift_pass_reads_the_old_key_while_the_move_is_pending(moto, tmp_path, monkeypatch): + """``fluid diff`` / ``verify --state-drift`` before any upgraded apply: the + real state, at the old key, with every resource in it, and nothing written. + Pointed at the new key instead, the pass found an empty state and said + ``not_checked``, a silent pass for a contract whose resources exist.""" + from fluid_build.cli import _diff_state + + contract_path = _write(tmp_path, _contract()) + _apply_as_the_old_release(contract_path, tmp_path) + old = _object(moto, _LEGACY_KEY) + before = _keys(moto) + monkeypatch.setenv("FLUID_STATE_BACKEND", f"s3://{_STATE_BUCKET}") + args = argparse.Namespace(provider=None, state_backend=None, workspace_dir=str(tmp_path)) + + report = _diff_state.check_state_drift( + yaml.safe_load(contract_path.read_text(encoding="utf-8")), args, _LOG + ) + + assert report.status == "checked", report.detail + assert report.state == f"remote: s3://{_STATE_BUCKET}/{_LEGACY_KEY}" + managed = [r for r in old["resources"] if r.get("mode") == "managed"] + assert len(report.resources) == len(managed) + assert not report.has_drift + assert _keys(moto) == before + + def test_the_data_loss_gate_still_closes_after_the_move(moto, tmp_path, monkeypatch): """The moved state is the one the gate judges: renaming the table plans a destroy, and without --allow-data-loss the apply refuses it.""" diff --git a/tests/iac/test_state_migration_safety.py b/tests/iac/test_state_migration_safety.py new file mode 100644 index 00000000..2cedd145 --- /dev/null +++ b/tests/iac/test_state_migration_safety.py @@ -0,0 +1,520 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The state move's safety branches, and what its scratch ``tofu init`` installs. + +Three branches of ``state_migration.reconcile_state_key`` had no test: a +mutant removing the re-check before the copy, one disabling the check after +it, and one pointing ``fluid diff`` at the new key all passed the suite. Here +the ``tofu`` calls are replaced by a scripted fake, so each branch is driven +on purpose: another job writing the new key between the first check and the +copy (``state_migration_raced``), a copy that reads back different resources +(``state_migration_unverified``), and a probe whose init failed before it +recorded the old backend (an error, never an empty state). + +The probe's scratch init used to install the providers the old state names, +at their latest version (OpenTofu installs what the state requires): a +``{"terraform": {}}`` module beside a state naming ``hashicorp/null`` +installed ``hashicorp/null v3.3.2``, and it ran on every apply whose new key +was empty. It now installs nothing (``-plugin-dir`` on an empty directory), +and the one-time move installs at the plugin's pins. The offline tests prove +the probe with real ``tofu`` and no registry reachable (a dead proxy): it +attributes the state; before, it reached for the registry and failed. + +Also here: the apply's refusal to plan on a key that names no provider when +it holds another cloud's resources, and ``fluid apply --dry-run`` asking the +move not to run. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from fluid_build.cli import _apply_opentofu_engine as engine +from fluid_build.cli._common import CLIError +from fluid_build.iac import runner +from fluid_build.iac import state_migration as mig +from fluid_build.iac.runner import TofuResult + +pytestmark = pytest.mark.unit + +_AWS = 'provider["registry.opentofu.org/hashicorp/aws"]' +_GOOGLE = 'provider["registry.opentofu.org/hashicorp/google"]' + + +def _state(lineage: str, *resources: Dict[str, Any]) -> Dict[str, Any]: + return { + "version": 4, + "terraform_version": "1.12.0", + "serial": 3, + "lineage": lineage, + "outputs": {}, + "resources": list(resources), + } + + +def _resource(rtype: str, name: str, provider: str) -> Dict[str, Any]: + return { + "mode": "managed", + "type": rtype, + "name": name, + "provider": provider, + # As `tofu state pull` prints it (it adds the empty list). + "instances": [ + {"schema_version": 0, "attributes": {"id": name}, "sensitive_attributes": []} + ], + } + + +_EMPTY = _state("") # what `tofu state pull` prints for a key with no state +_OLD = _state("11111111-aaaa", _resource("aws_s3_bucket", "lake", _AWS)) +_OTHER_JOB = _state("22222222-bbbb", _resource("aws_s3_bucket", "someone_else", _AWS)) + +_CURRENT = {"s3": {"bucket": "b", "key": "fluid/p/aws/terraform.tfstate"}} +_LEGACY = {"s3": {"bucket": "b", "key": "fluid/p/terraform.tfstate"}} + + +class _FakeTofu: + """Scripted ``tofu init`` / ``tofu state pull``, keyed by the directory's name. + + ``workdir`` is the apply's own directory (the new key), ``legacy`` the + probe's, ``legacy-plain`` its fallback and ``move`` the copy's. + """ + + def __init__(self, workdir_pulls: List[Dict[str, Any]], legacy: Dict[str, Any]) -> None: + self.workdir_pulls = list(workdir_pulls) + self.legacy = legacy + self.calls: List[Dict[str, Any]] = [] + #: The probe's -plugin-dir init: 0, or 1 as when it stops at the + #: provider step; ``records`` says whether it recorded the backend. + self.probe_init_rc = 0 + self.records = False + self.plain_init_rc = 0 + self.modules: Dict[str, Any] = {} + + def init(self, workdir: str, **kwargs: Any) -> TofuResult: + path = Path(workdir) + name = path.name + module = json.loads((path / "main.tf.json").read_text(encoding="utf-8")) + self.modules[f"{name}{'-copy' if kwargs.get('force_copy') else ''}"] = module + plugin_dir = kwargs.get("plugin_dir") + self.calls.append( + { + "init": name, + **kwargs, + "plugin_dir_empty": ( + plugin_dir is not None and not any(Path(plugin_dir).iterdir()) + ), + } + ) + rc = 0 + if name == "legacy": + rc = self.probe_init_rc + if self.records: + recorded = {"version": 3, "backend": {"type": "s3", "config": _LEGACY["s3"]}} + (path / ".terraform").mkdir(exist_ok=True) + (path / ".terraform" / "terraform.tfstate").write_text(json.dumps(recorded)) + elif name == "legacy-plain": + rc = self.plain_init_rc + return TofuResult("init", rc, "", "Error: Failed to query available provider packages") + + def pull(self, workdir: str, **_kwargs: Any) -> TofuResult: + name = Path(workdir).name + self.calls.append({"pull": name}) + doc = self.legacy if name.startswith("legacy") else self.workdir_pulls.pop(0) + return TofuResult("state-pull", 0, json.dumps(doc), "") + + @property + def copies(self) -> List[Dict[str, Any]]: + return [c for c in self.calls if c.get("force_copy")] + + +@pytest.fixture +def fake(monkeypatch): + def install(workdir_pulls, legacy=_OLD): + tofu = _FakeTofu(workdir_pulls, legacy) + monkeypatch.setattr(mig.runner, "tofu_init", tofu.init) + monkeypatch.setattr(mig.runner, "tofu_state_pull", tofu.pull) + return tofu + + return install + + +def _reconcile(tmp_path: Path, *, migrate: bool = True): + workdir = tmp_path / "workdir" + workdir.mkdir(exist_ok=True) + return mig.reconcile_state_key( + workdir=workdir, + current=_CURRENT, + legacy=_LEGACY, + provider="aws", + env={}, + migrate=migrate, + ) + + +# ── the re-check before the copy ───────────────────────────────────────── + + +def test_a_state_written_to_the_new_key_before_the_copy_is_never_overwritten(fake, tmp_path): + """Empty at the first look, another job's state at the second: refuse, copy nothing.""" + tofu = fake([_EMPTY, _OTHER_JOB]) + with pytest.raises(mig.StateMigrationError) as exc: + _reconcile(tmp_path) + assert exc.value.code == "state_migration_raced" + assert "nothing was moved" in str(exc.value) + assert tofu.copies == [] + + +def test_the_same_state_arriving_first_is_used_and_nothing_is_copied(fake, tmp_path): + """Another run of this apply moved it between the two looks: that is the state.""" + tofu = fake([_EMPTY, _state("33333333-cccc", *_OLD["resources"])]) + outcome = _reconcile(tmp_path) + assert outcome.outcome == mig.CURRENT + assert tofu.copies == [] + + +# ── the check after the copy ───────────────────────────────────────────── + + +def test_a_copy_that_reads_back_different_resources_is_an_error(fake, tmp_path): + tofu = fake([_EMPTY, _EMPTY, _OTHER_JOB]) + with pytest.raises(mig.StateMigrationError) as exc: + _reconcile(tmp_path) + assert exc.value.code == "state_migration_unverified" + assert "the old object is untouched" in str(exc.value) + # One copy, and nothing after it: no second attempt, no clean-up write. + assert len(tofu.copies) == 1 + assert tofu.calls[-1] == {"pull": "workdir"} + + +def test_a_copy_that_reads_back_the_same_resources_is_the_move(fake, tmp_path): + fake([_EMPTY, _EMPTY, _state("44444444-dddd", *_OLD["resources"])]) + outcome = _reconcile(tmp_path) + assert outcome.outcome == mig.MIGRATED + assert outcome.resources == 1 + + +# ── what the probe installs ────────────────────────────────────────────── + + +def test_the_probe_installs_nothing(fake, tmp_path): + """The probe's init takes providers from an empty directory only, never + from the apply's own ``.terraform/providers`` (whose entries a plugin + cache links; read as a mirror they broke the workdir).""" + (tmp_path / "workdir" / ".terraform" / "providers").mkdir(parents=True) + tofu = fake([_EMPTY]) + assert _reconcile(tmp_path, migrate=False).outcome == mig.PENDING + (probe,) = [c for c in tofu.calls if c.get("init") == "legacy"] + assert probe["plugin_dir_empty"] is True + assert ".terraform" not in probe["plugin_dir"] + assert [c for c in tofu.calls if "init" in c] == [probe] + + +def test_a_probe_stopped_at_the_provider_step_reads_the_recorded_backend(fake, tmp_path): + """The usual path for a state that names providers: init exits 1 after + recording the old backend, and the state is pulled from it.""" + tofu = fake([_EMPTY]) + tofu.probe_init_rc, tofu.records = 1, True + assert _reconcile(tmp_path, migrate=False).outcome == mig.PENDING + assert {"pull": "legacy"} in tofu.calls + assert not [c for c in tofu.calls if c.get("init") == "legacy-plain"] + + +def test_a_probe_that_recorded_no_backend_falls_back_to_a_plain_init(fake, tmp_path): + tofu = fake([_EMPTY]) + tofu.probe_init_rc = 1 + assert _reconcile(tmp_path, migrate=False).outcome == mig.PENDING + (plain,) = [c for c in tofu.calls if c.get("init") == "legacy-plain"] + assert plain.get("plugin_dir") is None + assert {"pull": "legacy"} not in tofu.calls # never read from an unrecorded backend + + +def test_a_probe_that_cannot_initialise_at_all_is_an_error_not_an_empty_state(fake, tmp_path): + tofu = fake([_EMPTY]) + tofu.probe_init_rc, tofu.plain_init_rc = 1, 1 + with pytest.raises(mig.StateMigrationError) as exc: + _reconcile(tmp_path, migrate=False) + assert exc.value.code == "state_migration_probe_failed" + assert not [c for c in tofu.calls if str(c.get("pull", "")).startswith("legacy")] + + +def test_the_move_installs_what_the_old_state_names_at_the_plugin_s_pins(fake, tmp_path): + """Once per contract, the copy's init installs providers: the pinned + ``~> 5.0`` aws, not the latest, and only what the state names.""" + tofu = fake([_EMPTY, _EMPTY, _state("55555555-eeee", *_OLD["resources"])]) + assert _reconcile(tmp_path).outcome == mig.MIGRATED + want = {"aws": {"source": "hashicorp/aws", "version": "~> 5.0"}} + assert tofu.modules["move"]["terraform"]["required_providers"] == want + assert tofu.modules["move"]["terraform"]["backend"] == _LEGACY + assert tofu.modules["move-copy"]["terraform"]["required_providers"] == want + assert tofu.modules["move-copy"]["terraform"]["backend"] == _CURRENT + + +def test_the_module_docstring_no_longer_claims_the_probe_downloads_nothing(): + assert "downloads nothing" not in (mig.__doc__ or "") + assert "-plugin-dir" in (mig.__doc__ or "") + + +# ── real tofu, no registry reachable ───────────────────────────────────── + +_TOFU = runner.tofu_path() +offline = pytest.mark.skipif( + _TOFU is None or sys.platform.startswith("win"), + reason="needs `tofu` on PATH (and a POSIX dead-proxy setup)", +) + + +def _offline_env(tmp_path: Path) -> Dict[str, str]: + """Every registry request fails fast: a dead proxy, no CLI config, no cache.""" + cli_config = tmp_path / "empty.tofurc" + cli_config.write_text("", encoding="utf-8") + env = {k: v for k, v in os.environ.items() if k != "TF_PLUGIN_CACHE_DIR"} + env.update( + { + "HTTPS_PROXY": "http://127.0.0.1:9", + "HTTP_PROXY": "http://127.0.0.1:9", + "NO_PROXY": "", + "TF_CLI_CONFIG_FILE": str(cli_config), + "TF_IN_AUTOMATION": "1", + } + ) + return env + + +def _local_workdir(tmp_path: Path, env: Dict[str, str]) -> Dict[str, Any]: + """The apply's workdir, initialised on a local "new key", as the apply leaves it.""" + workdir = tmp_path / "workdir" + workdir.mkdir() + current = {"local": {"path": str(tmp_path / "new" / "terraform.tfstate")}} + (workdir / "main.tf.json").write_text( + json.dumps({"terraform": {"backend": current}}), encoding="utf-8" + ) + assert runner.tofu_init(str(workdir), env=env).ok + return current + + +def _legacy_file(tmp_path: Path, doc: Dict[str, Any]) -> Dict[str, Any]: + path = tmp_path / "old" / "terraform.tfstate" + path.parent.mkdir() + path.write_text(json.dumps(doc), encoding="utf-8") + return {"local": {"path": str(path)}} + + +@offline +def test_the_probe_of_this_provider_s_state_reaches_no_registry(tmp_path): + env = _offline_env(tmp_path) + current = _local_workdir(tmp_path, env) + legacy = _legacy_file(tmp_path, _OLD) + + outcome = mig.reconcile_state_key( + workdir=tmp_path / "workdir", + current=current, + legacy=legacy, + provider="aws", + env=env, + migrate=False, + ) + + assert outcome.outcome == mig.PENDING + assert outcome.resources == 1 + assert not Path(current["local"]["path"]).exists() + assert json.loads(Path(legacy["local"]["path"]).read_text(encoding="utf-8")) == _OLD + + +@offline +def test_another_cloud_s_state_is_attributed_with_nothing_installed(tmp_path): + """The gcp apply reading the aws state at the old key: the aws provider is + not fetched to find out whose state it is.""" + env = _offline_env(tmp_path) + current = _local_workdir(tmp_path, env) + legacy = _legacy_file(tmp_path, _OLD) + + outcome = mig.reconcile_state_key( + workdir=tmp_path / "workdir", + current=current, + legacy=legacy, + provider="gcp", + env=env, + migrate=True, + ) + + assert outcome.outcome == mig.OTHER_PROVIDER + assert "aws" in outcome.detail + assert not Path(current["local"]["path"]).exists() + + +# ── one state, two clouds, on a key that names no provider ─────────────── + + +def _target(tmp_path: Path, key: str) -> engine.StateTarget: + return engine.StateTarget( + workdir=tmp_path, backend={"s3": {"bucket": "b", "key": key}}, origin="--state-backend" + ) + + +def _stub_state(monkeypatch, doc: Dict[str, Any]) -> List[Path]: + seen: List[Path] = [] + + def read_state(workdir, env): + seen.append(Path(workdir)) + return mig.parse_state(json.dumps(doc)) + + monkeypatch.setattr(engine, "read_state", read_state) + return seen + + +def test_the_shared_key_holding_the_other_cloud_s_resources_is_refused(tmp_path, monkeypatch): + _stub_state(monkeypatch, _OLD) + with pytest.raises(CLIError) as exc: + engine.guard_state_shared_with_another_cloud( + _target(tmp_path, "fluid/terraform.tfstate"), "gcp", {} + ) + assert exc.value.event == "state_shared_with_another_provider" + assert exc.value.context["providers"] == ["aws"] + assert exc.value.context["state"] == "s3://b/fluid/terraform.tfstate" + + +@pytest.mark.parametrize( + "doc", + [_OLD, _EMPTY, _state("x", _resource("null_resource", "n", 'provider["x/hashicorp/null"]'))], +) +def test_this_cloud_s_own_or_no_one_s_resources_pass(tmp_path, monkeypatch, doc): + _stub_state(monkeypatch, doc) + engine.guard_state_shared_with_another_cloud( + _target(tmp_path, "fluid/terraform.tfstate"), "aws", {} + ) + + +def test_a_key_that_names_the_provider_is_not_read(tmp_path, monkeypatch): + seen = _stub_state(monkeypatch, _state("y", _resource("g", "d", _GOOGLE))) + engine.guard_state_shared_with_another_cloud( + _target(tmp_path, "fluid/p/aws/terraform.tfstate"), "aws", {} + ) + local = engine.StateTarget(workdir=tmp_path, backend=None, origin="default") + engine.guard_state_shared_with_another_cloud(local, "aws", {}) + assert seen == [] + + +def test_the_bucket_only_flag_route_is_refused_end_to_end(tmp_path, monkeypatch): + """--state-backend s3:// and a contract without packaging: both + clouds resolve to fluid/terraform.tfstate, and the gcp apply finds aws there.""" + monkeypatch.delenv("FLUID_STATE_BACKEND", raising=False) + contract = {"id": "bronze.customer_subscriptions", "name": "Customer Subscriptions"} + args = argparse.Namespace( + state_backend="s3://fluid-demo-lab-state", workspace_dir=str(tmp_path) + ) + aws = engine.resolve_state_target(args, contract, "aws") + gcp = engine.resolve_state_target(args, contract, "gcp") + assert aws.backend == gcp.backend # the shared legacy key, unchanged + _stub_state(monkeypatch, _OLD) + with pytest.raises(CLIError) as exc: + engine.guard_state_shared_with_another_cloud(gcp, "gcp", {}) + assert exc.value.event == "state_shared_with_another_provider" + + +# ── fluid apply --dry-run never moves state ────────────────────────────── + + +class _Stop(Exception): + pass + + +def _engine_until_the_guard(monkeypatch, tmp_path: Path, *, dry_run: bool) -> Dict[str, Any]: + """Run the engine with tofu stubbed, up to the shared-state guard.""" + seen: Dict[str, Any] = {"inits": []} + contract = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "bronze.customer_subscriptions", + "name": "Customer Subscriptions", + "metadata": {"owner": {"team": "t"}}, + "exposes": [ + { + "exposeId": "subscriptions", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": {"bucket": "lake", "path": "bronze/", "database": "d"}, + }, + "contract": {"schema": [{"name": "a", "type": "VARCHAR"}]}, + } + ], + } + monkeypatch.setenv("FLUID_STATE_BACKEND", "s3://state-bucket") + monkeypatch.setattr(engine, "_verify_plan_binding_for_opentofu", lambda *a, **k: None) + monkeypatch.setattr(engine, "_load_contract", lambda *a, **k: contract) + monkeypatch.setattr(engine, "native_actions", lambda *a, **k: []) + monkeypatch.setattr(engine.runner, "tofu_path", lambda: "/usr/bin/tofu") + monkeypatch.setattr(engine.runner, "require_tofu_version", lambda *a, **k: None) + monkeypatch.setattr(engine, "cprint", lambda *a, **k: None) + + def init(workdir, **kwargs): + module = json.loads((Path(workdir) / "main.tf.json").read_text(encoding="utf-8")) + seen["inits"].append((module["terraform"]["backend"]["s3"]["key"], kwargs)) + return TofuResult("init", 0, "", "") + + def reconcile(**kwargs): + seen["migrate"] = kwargs["migrate"] + return mig.StateReconciliation(mig.PENDING, kwargs["legacy"], kwargs["current"], 1) + + def stop(target, provider, env): + seen["guarded"] = engine.backend_location(target.backend) + raise _Stop + + monkeypatch.setattr(engine.runner, "tofu_init", init) + monkeypatch.setattr(engine, "_reconcile_state", reconcile) + monkeypatch.setattr(engine, "guard_state_shared_with_another_cloud", stop) + args = argparse.Namespace( + contract="c.fluid.yaml", + env=None, + provider="aws", + workspace_dir=str(tmp_path), + state_backend=None, + dry_run=dry_run, + allow_data_loss=False, + no_verify_plan_binding=True, + ) + with pytest.raises(_Stop): + engine.apply_via_opentofu(args, logging.getLogger("test.state_migration_safety")) + return seen + + +def test_a_dry_run_plans_on_the_old_key_and_never_asks_for_the_move(monkeypatch, tmp_path): + seen = _engine_until_the_guard(monkeypatch, tmp_path, dry_run=True) + assert seen["migrate"] is False + # Initialised on the new key first, then re-pointed at the old one. + assert [key for key, _ in seen["inits"]] == [ + "fluid/bronze.customer_subscriptions/aws/terraform.tfstate", + "fluid/bronze.customer_subscriptions/terraform.tfstate", + ] + assert seen["inits"][1][1].get("reconfigure") is True + assert ( + seen["guarded"] == "s3://state-bucket/fluid/bronze.customer_subscriptions/terraform.tfstate" + ) + + +def test_a_real_apply_asks_for_the_move(monkeypatch, tmp_path): + seen = _engine_until_the_guard(monkeypatch, tmp_path, dry_run=False) + assert seen["migrate"] is True + assert len(seen["inits"]) == 1 diff --git a/tests/policy/test_sovereignty_enforcement_mode.py b/tests/policy/test_sovereignty_enforcement_mode.py index a5c7d0eb..621b7f48 100644 --- a/tests/policy/test_sovereignty_enforcement_mode.py +++ b/tests/policy/test_sovereignty_enforcement_mode.py @@ -240,18 +240,53 @@ def test_catch_all_jurisdiction_is_not_a_violation_anywhere(jurisdiction: str) - assert SovereigntyValidator().validate(doc)[0] is True -def test_unknown_region_does_not_block_under_strict() -> None: - """An unmappable region is 'cannot tell', not 'violates'. - - Deliberately unchanged. Failing closed here would be defensible for a - sovereignty control, but it would break every contract using a region the - vendored table does not carry, so it is a separate decision rather than a - side effect of this fix. +def test_unknown_region_on_a_cloud_blocks_under_strict() -> None: + """An unmappable cloud region under a strict jurisdiction is refused. + + This used to be a warning ("cannot tell" is not "violates"), which failed + open: the GCP table lagged Google by nine regions and the ASIA + multi-region, so an EU-only strict contract validated and emitted + me-central2. On an aws / gcp / azure binding strict now refuses a + jurisdiction it cannot show; advisory still warns. """ doc = contract(region="mars-central-1", enforcementMode="strict", jurisdiction="EU") is_valid, violations = SovereigntyValidator().validate(doc) + assert is_valid is False + (finding,) = _jurisdiction_findings(violations) + assert finding.severity == "error" + assert "sovereignty.allowedRegions" in (finding.suggestion or "") + + advisory = contract(region="mars-central-1", enforcementMode="advisory", jurisdiction="EU") + is_valid, violations = SovereigntyValidator().validate(advisory) + assert is_valid is True + assert [v.severity for v in _jurisdiction_findings(violations)] == ["warning"] + + +def _jurisdiction_findings(violations: List[Any]) -> List[Any]: + """Check 3's findings (check 4 adds its own "no known jurisdiction" warning).""" + return [v for v in violations if "does not match required jurisdiction" in v.message] + + +def test_an_unknown_region_named_in_allowed_regions_still_only_warns() -> None: + """The operator vouched for it: the one way to use a region the table lacks.""" + doc = contract( + region="mars-central-1", + enforcementMode="strict", + jurisdiction="EU", + allowedRegions=["mars-central-1"], + ) + is_valid, violations = SovereigntyValidator().validate(doc) + assert is_valid is True + assert [v.severity for v in _jurisdiction_findings(violations)] == ["warning"] + + +def test_an_unknown_region_off_the_clouds_still_only_warns() -> None: + """A platform whose region is not a cloud region keeps the warning.""" + doc = contract(region="mars-central-1", enforcementMode="strict", jurisdiction="EU") + doc["exposes"][0]["binding"]["platform"] = "snowflake" + is_valid, violations = SovereigntyValidator().validate(doc) assert is_valid is True - assert any(v.severity == "warning" for v in violations) + assert [v.severity for v in _jurisdiction_findings(violations)] == ["warning"] def test_explicit_deny_is_an_error_in_every_mode() -> None: diff --git a/tests/providers/test_gcp_locations_and_pubsub_placement.py b/tests/providers/test_gcp_locations_and_pubsub_placement.py new file mode 100644 index 00000000..a40585c5 --- /dev/null +++ b/tests/providers/test_gcp_locations_and_pubsub_placement.py @@ -0,0 +1,280 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GCP sovereignty fails closed on a location the table cannot place, and a +Pub/Sub topic keeps its region. + +Measured on the branch before this change, with ``sovereignty: {jurisdiction: +EU, enforcementMode: strict}`` and no ``allowedRegions``: a gcp binding in +``me-central2``, ``northamerica-south1`` or the ``asia`` multi-region gave +``fluid validate`` rc 0 (a warning, "jurisdiction: Unknown") and ``fluid +generate iac`` rc 0 with that location in the module. The vendored region +table (dgl/cloud-regions) lacks nine of Google's regions, among them three EU +ones (europe-north2, europe-west10, europe-west12), and no multi- or +dual-region but US and EU. + +A gcp ``pubsub_topic`` binding's ``location.region`` was dropped: the emitted +``google_pubsub_topic`` had no ``message_storage_policy``, so messages could +be stored outside an ``allowedRegions: [europe-west1]`` policy that validate +and generate had both passed. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional + +import pytest +import yaml + +from fluid_build._errors import SovereigntyViolationError +from fluid_build.iac import get_iac_plugin +from fluid_build.policy.sovereignty import SovereigntyValidator, region_jurisdiction_map +from fluid_build.providers.gcp.util.sovereignty import resource_placements + +pytestmark = pytest.mark.unit + +#: BigQuery's region list (docs.cloud.google.com/bigquery/docs/locations, +#: read 2026-09-28), with the country each is in. +_GOOGLE_REGIONS = { + "us-east5": "US", + "us-south1": "US", + "us-central1": "US", + "us-west2": "US", + "us-west4": "US", + "northamerica-south1": "MX", + "northamerica-northeast1": "CA", + "us-east4": "US", + "us-central2": "US", + "us-west1": "US", + "us-west3": "US", + "southamerica-east1": "BR", + "southamerica-west1": "CL", + "us-east1": "US", + "northamerica-northeast2": "CA", + "asia-southeast3": "TH", + "asia-south2": "IN", + "asia-east2": "HK", + "asia-southeast2": "ID", + "australia-southeast2": "AU", + "asia-south1": "IN", + "asia-northeast2": "JP", + "asia-northeast3": "KR", + "asia-southeast1": "SG", + "australia-southeast1": "AU", + "asia-east1": "TW", + "asia-northeast1": "JP", + "europe-west1": "EU", + "europe-west10": "EU", + "europe-north1": "EU", + "europe-west3": "EU", + "europe-west2": "UK", + "europe-southwest1": "EU", + "europe-west8": "EU", + "europe-west4": "EU", + "europe-west9": "EU", + "europe-north2": "EU", + "europe-west12": "EU", + "europe-central2": "EU", + "europe-west6": "CH", + "me-central2": "SA", + "me-central1": "QA", + "me-west1": "IL", + "africa-south1": "ZA", +} + + +@pytest.mark.parametrize("region, jurisdiction", sorted(_GOOGLE_REGIONS.items())) +def test_every_bigquery_region_has_its_jurisdiction(region, jurisdiction): + assert region_jurisdiction_map().get(region) == jurisdiction + + +@pytest.mark.parametrize( + "location, jurisdiction", + [("EUR4", "EU"), ("eur4", "EU"), ("NAM4", "US"), ("ASIA1", "JP"), ("EU", "EU"), ("US", "US")], +) +def test_a_multi_or_dual_region_within_one_jurisdiction_resolves(location, jurisdiction): + assert region_jurisdiction_map().get(location) == jurisdiction + + +@pytest.mark.parametrize("location", ["asia", "ASIA", "EUR5", "EUR7", "eur8"]) +def test_a_location_spanning_jurisdictions_stays_unknown(location): + """ASIA spans several countries; EUR5/EUR7/EUR8 each pair an EU region + with London or Zürich.""" + assert region_jurisdiction_map().get(location) is None + + +_EU_STRICT = {"jurisdiction": "EU", "enforcementMode": "strict"} + + +def _contract( + location: Dict[str, Any], + *, + fmt: str = "bigquery_table", + sovereignty: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + doc: Dict[str, Any] = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "bronze.unknown_probe", + "name": "Unknown Probe", + "domain": "Customer", + "metadata": {"layer": "Bronze", "owner": {"team": "data-platform"}}, + "exposes": [ + { + "exposeId": "t", + "kind": "table", + "binding": {"platform": "gcp", "format": fmt, "location": location}, + "contract": {"schema": [{"name": "a", "type": "STRING"}]}, + } + ], + } + if sovereignty is not None: + doc["sovereignty"] = sovereignty + return doc + + +def _bq(region: str) -> Dict[str, Any]: + return {"project": "p", "dataset": "d", "table": "t", "region": region} + + +def _emit(contract: Dict[str, Any]) -> Dict[str, Any]: + return get_iac_plugin("gcp").emit(contract, []) + + +@pytest.mark.parametrize("region", ["me-central2", "northamerica-south1", "asia", "EUR5"]) +def test_a_strict_eu_policy_refuses_a_location_it_cannot_place(region): + ok, violations = SovereigntyValidator().validate(_contract(_bq(region), sovereignty=_EU_STRICT)) + assert ok is False + assert any( + v.severity == "error" and "does not match required jurisdiction" in v.message + for v in violations + ) + with pytest.raises(SovereigntyViolationError): + _emit(_contract(_bq(region), sovereignty=_EU_STRICT)) + + +@pytest.mark.parametrize("region", ["europe-west10", "europe-west12", "europe-north2", "EUR4"]) +def test_an_eu_location_the_vendored_table_lacked_now_passes_cleanly(region): + ok, violations = SovereigntyValidator().validate(_contract(_bq(region), sovereignty=_EU_STRICT)) + assert ok is True + assert not [v for v in violations if "jurisdiction" in v.message] + (dataset,) = _emit(_contract(_bq(region), sovereignty=_EU_STRICT))[ + "google_bigquery_dataset" + ].values() + assert dataset["location"] == region + + +def test_advisory_still_only_warns_about_a_location_it_cannot_place(): + policy = dict(_EU_STRICT, enforcementMode="advisory") + ok, _ = SovereigntyValidator().validate(_contract(_bq("asia"), sovereignty=policy)) + assert ok is True + _emit(_contract(_bq("asia"), sovereignty=policy)) + + +# ── Pub/Sub: the binding's region is where messages may be stored ─────── + + +def _topic(region: Optional[str] = "europe-west1") -> Dict[str, Any]: + loc: Dict[str, Any] = {"project": "p", "topic": "customer-events"} + if region: + loc["region"] = region + return loc + + +_PINNED = {"jurisdiction": "EU", "allowedRegions": ["europe-west1"], "enforcementMode": "strict"} + + +def test_a_pubsub_binding_s_region_becomes_its_message_storage_policy(): + resources = _emit(_contract(_topic(), fmt="pubsub_topic", sovereignty=_PINNED)) + (topic,) = resources["google_pubsub_topic"].values() + assert topic["message_storage_policy"] == {"allowed_persistence_regions": ["europe-west1"]} + + +def test_without_a_policy_the_region_is_applied_too(): + """Never dropped: the field is the platform's to apply, policy or not.""" + (topic,) = _emit(_contract(_topic(), fmt="pubsub_topic"))["google_pubsub_topic"].values() + assert topic["message_storage_policy"]["allowed_persistence_regions"] == ["europe-west1"] + + +def test_a_topic_with_no_region_has_no_storage_policy_and_no_policy_passes(): + (topic,) = _emit(_contract(_topic(None), fmt="pubsub_topic"))["google_pubsub_topic"].values() + assert "message_storage_policy" not in topic + + +def test_a_pubsub_region_outside_the_policy_is_refused(): + with pytest.raises(SovereigntyViolationError) as exc: + _emit(_contract(_topic("us-central1"), fmt="pubsub_topic", sovereignty=_PINNED)) + assert "us-central1" in exc.value.what + + +def test_a_pubsub_binding_with_no_region_is_refused_under_strict(): + with pytest.raises(SovereigntyViolationError) as exc: + _emit(_contract(_topic(None), fmt="pubsub_topic", sovereignty=_PINNED)) + assert "declares no region" in exc.value.what + + +def test_the_hook_reads_the_persistence_regions_back(): + resources = { + "google_pubsub_topic": { + "a": {"name": "a", "message_storage_policy": {"allowed_persistence_regions": ["x1"]}}, + "b": { + "name": "b", + "message_storage_policy": [{"allowed_persistence_regions": ["y1", "${var.r}"]}], + }, + } + } + assert resource_placements(resources) == [ + ("google_pubsub_topic.a", "x1"), + ("google_pubsub_topic.b", "y1"), + ] + + +# ── Through the real CLI ─────────────────────────────────────────────────── + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch) -> Path: + for var in ("FLUID_PROVIDER", "FLUID_REGION", "GOOGLE_APPLICATION_CREDENTIALS"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _cli(*argv: str) -> int: + from fluid_build.cli import main + + return main(list(argv)) + + +def _write(root: Path, contract: Dict[str, Any]) -> Path: + path = root / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + return path + + +def test_validate_and_generate_refuse_me_central2_under_a_strict_eu_policy(workspace): + path = _write(workspace, _contract(_bq("me-central2"), sovereignty=_EU_STRICT)) + assert _cli("validate", str(path)) == 1 + assert _cli("generate", "iac", str(path), "--out", str(workspace / "iac")) != 0 + assert not (workspace / "iac" / "main.tf.json").exists() + + +def test_generate_writes_the_topic_s_storage_policy(workspace): + path = _write(workspace, _contract(_topic(), fmt="pubsub_topic", sovereignty=_PINNED)) + assert _cli("validate", str(path)) == 0 + assert _cli("generate", "iac", str(path), "--out", str(workspace / "iac")) == 0 + module = (workspace / "iac" / "main.tf.json").read_text(encoding="utf-8") + assert '"allowed_persistence_regions"' in module From fbc31fd97330251ac7bd993e7002b0b670c75a05 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 15:46:39 +0200 Subject: [PATCH 03/10] fix(sovereignty): a KMS key ring and a Data Catalog taxonomy are placed at their dataset's multi-region, and plan --check-sovereignty checks what apply emits Cloud KMS names BigQuery's EU and US multi-regions europe and us, and Data Catalog names them eu and us. The GCP sovereignty hook checked those ids as places, so with the governance branch's CMEK and policy tags an EU-only strict contract with an EU dataset was refused at apply (Region 'europe' not in allowed regions list, jurisdiction Unknown), after validate --strict and plan --check-sovereignty passed. resource_placements now places both at the multi-region, spelled as the emitted dataset spells it. fluid plan --check-sovereignty ran only the policy engine over the bindings for gcp (the provider had no hook), so a placement the planner or the emitter derived passed stage 6 and was refused at stage 7. GcpProvider now has validate_sovereignty: the engine's contract checks plus the planned actions' and the emitted resources' placements, through the same engine, and it returns the error-severity findings. GcpIacPlugin.emit takes enforce_sovereignty=False for it. A hook that gives no verdict is now named as such in the fallback line, not as a missing hook. --- fluid_build/cli/plan.py | 15 +- fluid_build/iac/providers/gcp.py | 15 +- fluid_build/providers/gcp/provider.py | 51 +++++++ fluid_build/providers/gcp/util/sovereignty.py | 43 +++++- .../test_gcp_sovereignty_fail_closed.py | 139 ++++++++++++++++++ 5 files changed, 253 insertions(+), 10 deletions(-) diff --git a/fluid_build/cli/plan.py b/fluid_build/cli/plan.py index 2295fc4b..56d5cf5a 100644 --- a/fluid_build/cli/plan.py +++ b/fluid_build/cli/plan.py @@ -602,11 +602,16 @@ def _report_sovereignty( return False # No usable provider verdict — fall back to the built-in policy engine. - reason = ( - f"the {pname} provider has no sovereignty hook" - if hook_provider is not None - else "no provider could be built for this contract" - ) + if hook_provider is None: + reason = "no provider could be built for this contract" + else: + from fluid_build.cli.hooks import has_hook + + reason = ( + f"the {pname} provider's sovereignty hook gave no verdict" + if has_hook(hook_provider, "validate_sovereignty") + else f"the {pname} provider has no sovereignty hook" + ) sovereignty = contract.get("sovereignty") or {} if not isinstance(sovereignty, dict) or not sovereignty: diff --git a/fluid_build/iac/providers/gcp.py b/fluid_build/iac/providers/gcp.py index 0bc291d2..9ea6ef45 100644 --- a/fluid_build/iac/providers/gcp.py +++ b/fluid_build/iac/providers/gcp.py @@ -329,8 +329,18 @@ class GcpIacPlugin: ) def emit( - self, contract: Mapping[str, Any], actions: Iterable[Mapping[str, Any]] = () + self, + contract: Mapping[str, Any], + actions: Iterable[Mapping[str, Any]] = (), + *, + enforce_sovereignty: bool = True, ) -> Dict[str, Any]: + """The contract's GCP resources, refused when they land outside its sovereignty. + + ``enforce_sovereignty=False`` returns them unchecked, for a caller that + runs the same check itself and reports it (``GcpProvider.validate_sovereignty``, + what ``fluid plan --check-sovereignty`` reads). + """ resources: Dict[str, Dict[str, Any]] = {} cid = safe_ident(contract.get("id") or contract.get("name") or "product") base_labels = {"managed_by": "fluid", "fluid_contract": cid} @@ -398,7 +408,8 @@ def emit( resource_placements, ) - enforce_gcp_sovereignty(contract, resource_placements(resources)) + if enforce_sovereignty: + enforce_gcp_sovereignty(contract, resource_placements(resources)) return resources def emit_data( diff --git a/fluid_build/providers/gcp/provider.py b/fluid_build/providers/gcp/provider.py index 8bdf6196..3ebb00fc 100644 --- a/fluid_build/providers/gcp/provider.py +++ b/fluid_build/providers/gcp/provider.py @@ -231,6 +231,57 @@ def _validate_sovereignty( raise ProviderError(str(e)) from e self.info_kv(event="sovereignty_validated", region=self.region) + def validate_sovereignty(self, contract: Mapping[str, Any]) -> Optional[List[str]]: + """``fluid plan --check-sovereignty``: what ``fluid apply`` would refuse, and why. + + Stage 7 checks sovereignty twice, both where the data lands: this + provider's planned actions (:meth:`_validate_sovereignty`) and every + resource the OpenTofu plugin emits (``GcpIacPlugin.emit``: the dataset, + and the KMS key ring and Data Catalog taxonomy governance adds). The + plan stage used to run only the policy engine over the bindings, so a + placement the emitter derived passed stage 6 and was refused at stage + 7. This runs the same placements through the same engine, plus the + engine's own contract-level checks, and returns every error-severity + finding (``where: message``); a warning or an info finding is logged, + as apply logs it, and does not block. + + ``None`` (no verdict) for a contract with no ``sovereignty`` block, so + the plan reports NOT CHECKED rather than a pass. A planner or emitter + that cannot run raises; ``run_validate_sovereignty`` reads that as no + verdict too, and the plan falls back to the policy engine. + """ + if not contract.get("sovereignty"): + return None + from fluid_build.iac import get_iac_plugin + from fluid_build.iac.plan_packaging import filter_referenced_container_actions + from fluid_build.policy.sovereignty import SovereigntyValidator + + from .util.sovereignty import ( + action_placements, + gcp_sovereignty_violations, + resource_placements, + ) + + actions = plan_actions(contract, self.project, self.region, self.logger) + # What apply emits from: the planned actions less the creations a + # shared pool already holds (``generate_iac.native_actions``). + kept, _dropped = filter_referenced_container_actions(contract, list(actions)) + resources = get_iac_plugin("gcp").emit(contract, kept, enforce_sovereignty=False) + + _ok, found = SovereigntyValidator().validate(dict(contract)) + found = list(found) + found += gcp_sovereignty_violations(contract, action_placements(actions)) + found += gcp_sovereignty_violations(contract, resource_placements(resources)) + errors: List[str] = [] + for v in found: + line = f"{v.expose_id}: {v.message}" + if v.severity == "error": + if line not in errors: + errors.append(line) + else: + self.warn_kv(event="sovereignty_finding", severity=v.severity, finding=line) + return errors + def apply(self, actions: List[Dict[str, Any]], **kwargs: Any) -> ApplyResult: """Native GCP apply is retired — GCP uses the OpenTofu engine. diff --git a/fluid_build/providers/gcp/util/sovereignty.py b/fluid_build/providers/gcp/util/sovereignty.py index 415d55dc..cd943ef6 100644 --- a/fluid_build/providers/gcp/util/sovereignty.py +++ b/fluid_build/providers/gcp/util/sovereignty.py @@ -45,7 +45,7 @@ from __future__ import annotations import logging -from typing import Any, Iterable, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from fluid_build.policy.sovereignty import ( SovereigntyValidator, @@ -74,6 +74,21 @@ def unplaced_gcp_exposes(contract: Mapping[str, Any]) -> List[Placement]: return out +#: Services that name BigQuery's two multi-regions in their own vocabulary, by +#: resource type: ``{their location id: the BigQuery multi-region}``. A key +#: for a dataset in ``EU`` must be in the Cloud KMS location ``europe`` +#: (``us`` for ``US``; the BigQuery CMEK guide), and the taxonomy whose policy +#: tags restrict its columns in the Data Catalog location ``eu`` (``us``). +#: Read as themselves, those ids failed ``allowedRegions: [EU]`` and, under a +#: strict jurisdiction, resolved to none, so the key ring and the taxonomy of +#: an EU dataset were refused at apply after validate and plan passed. They +#: are the dataset's own place, and are checked as it. +_BIGQUERY_MULTI_REGION_ALIASES: Mapping[str, Mapping[str, str]] = { + "google_kms_key_ring": {"europe": "eu", "us": "us"}, + "google_data_catalog_taxonomy": {"eu": "eu", "us": "us"}, +} + + def resource_placements(resources: Mapping[str, Any]) -> List[Placement]: """``(address, location)`` for every emitted resource that names one. @@ -81,8 +96,11 @@ def resource_placements(resources: Mapping[str, Any]) -> List[Placement]: OpenTofu reference (``${...}``) is not a place and is skipped. A Pub/Sub topic has no ``location``: where its messages are stored is its ``message_storage_policy.allowed_persistence_regions``, one placement - per region. + per region. A KMS key ring or a Data Catalog taxonomy in one of + BigQuery's multi-regions is placed at that multi-region, spelled as the + emitted dataset spells it (:data:`_BIGQUERY_MULTI_REGION_ALIASES`). """ + spellings = _multi_region_spellings(resources) out: List[Placement] = [] for rtype, by_name in (resources or {}).items(): if not isinstance(by_name, Mapping): @@ -93,13 +111,32 @@ def resource_placements(resources: Mapping[str, Any]) -> List[Placement]: for key in ("location", "region"): value = body.get(key) if _is_place(value): - out.append((f"{rtype}.{name}", str(value))) + out.append((f"{rtype}.{name}", _as_placed(rtype, str(value), spellings))) break for region in _persistence_regions(body): out.append((f"{rtype}.{name}", region)) return out +def _multi_region_spellings(resources: Mapping[str, Any]) -> Dict[str, str]: + """``{"eu" | "us": location}`` as the emitted BigQuery datasets spell each multi-region.""" + out: Dict[str, str] = {} + datasets = (resources or {}).get("google_bigquery_dataset") + for body in (datasets or {}).values() if isinstance(datasets, Mapping) else (): + value = body.get("location") if isinstance(body, Mapping) else None + if _is_place(value) and str(value).lower() in ("eu", "us"): + out.setdefault(str(value).lower(), str(value)) + return out + + +def _as_placed(rtype: str, value: str, spellings: Mapping[str, str]) -> str: + """``value``, or the BigQuery multi-region it names for ``rtype``.""" + multi = _BIGQUERY_MULTI_REGION_ALIASES.get(rtype, {}).get(value.lower()) + if multi is None: + return value + return spellings.get(multi, multi.upper()) + + def _is_place(value: Any) -> bool: return isinstance(value, str) and bool(value) and not value.startswith("${") diff --git a/tests/providers/test_gcp_sovereignty_fail_closed.py b/tests/providers/test_gcp_sovereignty_fail_closed.py index b8fb54ee..70a6aab3 100644 --- a/tests/providers/test_gcp_sovereignty_fail_closed.py +++ b/tests/providers/test_gcp_sovereignty_fail_closed.py @@ -331,3 +331,142 @@ class WriteDisposition: str(landed), target, mode=mode, sink_format=sink, expected_rows=1, logger=_LOG ) assert calls == ["europe-west1"] + + +# ── A key ring and a taxonomy in a BigQuery multi-region ────────────────── +# +# Measured on the integration of this branch with the governance one: a +# contract with ``allowedRegions: [EU]`` and an ``EU`` dataset emits its CMEK +# key ring in the Cloud KMS location ``europe`` and its policy-tag taxonomy in +# the Data Catalog location ``eu``. ``fluid validate --strict`` and ``fluid plan +# --check-sovereignty`` passed, and ``fluid generate iac`` / ``fluid apply`` +# refused both ("Region 'europe' not in allowed regions list", "(jurisdiction: +# Unknown)"). They are the dataset's own place. + +_EU_ONLY = {"jurisdiction": "EU", "allowedRegions": ["EU"], "enforcementMode": "strict"} + + +def _multi_region_contract(region: str) -> Dict[str, Any]: + contract = _contract(dict(_BQ, region=region)) + contract["sovereignty"] = dict(_EU_ONLY) + return contract + + +def _governed(dataset_location: str, ring: str, taxonomy: str) -> Dict[str, Any]: + return { + "google_bigquery_dataset": {"d": {"location": dataset_location}}, + "google_kms_key_ring": {"k": {"name": "ring", "location": ring}}, + "google_data_catalog_taxonomy": {"t": {"display_name": "tax", "region": taxonomy}}, + } + + +def test_a_key_ring_and_a_taxonomy_are_placed_at_their_datasets_multi_region(): + from fluid_build.providers.gcp.util.sovereignty import resource_placements + + assert resource_placements(_governed("EU", "europe", "eu")) == [ + ("google_bigquery_dataset.d", "EU"), + ("google_kms_key_ring.k", "EU"), + ("google_data_catalog_taxonomy.t", "EU"), + ] + assert resource_placements(_governed("US", "us", "us"))[1:] == [ + ("google_kms_key_ring.k", "US"), + ("google_data_catalog_taxonomy.t", "US"), + ] + # Spelled as the dataset spells it, so ``allowedRegions: [eu]`` agrees too. + assert {p for _, p in resource_placements(_governed("eu", "europe", "eu"))} == {"eu"} + # A regional key ring or taxonomy is where it says. + assert resource_placements(_governed("europe-west1", "europe-west1", "europe-west1")) == [ + ("google_bigquery_dataset.d", "europe-west1"), + ("google_kms_key_ring.k", "europe-west1"), + ("google_data_catalog_taxonomy.t", "europe-west1"), + ] + + +def test_an_eu_datasets_key_ring_and_taxonomy_pass_an_eu_only_strict_policy(): + from fluid_build.providers.gcp.util.sovereignty import ( + enforce_gcp_sovereignty, + resource_placements, + ) + + contract = _multi_region_contract("EU") + enforce_gcp_sovereignty(contract, resource_placements(_governed("EU", "europe", "eu"))) + # With only a jurisdiction, ``europe`` used to resolve to none (strict refuses). + contract["sovereignty"] = {"jurisdiction": "EU", "enforcementMode": "strict"} + enforce_gcp_sovereignty(contract, resource_placements(_governed("EU", "europe", "eu"))) + + +def test_a_us_datasets_key_ring_and_taxonomy_are_still_refused_under_an_eu_policy(): + from fluid_build.providers.gcp.util.sovereignty import ( + enforce_gcp_sovereignty, + resource_placements, + ) + + with pytest.raises(SovereigntyViolationError) as exc: + enforce_gcp_sovereignty( + _multi_region_contract("US"), resource_placements(_governed("US", "us", "us")) + ) + assert "google_kms_key_ring.k: Region 'US' not in allowed regions list" in exc.value.what + assert "(jurisdiction: US)" in exc.value.what + + +# ── fluid plan --check-sovereignty runs what fluid apply runs ───────────── + + +def _hook(contract: Dict[str, Any]): + from fluid_build.providers.gcp.provider import GcpProvider + + return GcpProvider(project="northwind-demo", region="europe-west1").validate_sovereignty( + contract + ) + + +def test_the_plan_hook_checks_the_resources_the_emitter_places(monkeypatch): + """A place only the emitter derives is refused at stage 6, as at stage 7.""" + from fluid_build.iac.providers.gcp import GcpIacPlugin + + real_emit = GcpIacPlugin.emit + + def emit_with_a_key_ring(self, contract, actions=(), **kwargs): + resources = real_emit(self, contract, actions, **kwargs) + resources["google_kms_key_ring"] = {"k": {"name": "ring", "location": "asia"}} + return resources + + monkeypatch.setattr(GcpIacPlugin, "emit", emit_with_a_key_ring) + contract = _contract(dict(_BQ, region="europe-west1")) + errors = _hook(contract) + assert errors and all(e.startswith("google_kms_key_ring.k: ") for e in errors) + assert any("Region 'asia' not in allowed regions list" in e for e in errors) + + +@pytest.mark.parametrize( + "location, mode", + [ + (dict(_BQ), "strict"), + (dict(_BQ, region="us-central1"), "strict"), + (dict(_BQ, location="us-central1"), "strict"), + (dict(_BQ, region="europe-west1"), "strict"), + (dict(_BQ), "advisory"), + (dict(_BQ, region="us-central1"), "audit"), + ], +) +def test_the_plan_hook_refuses_exactly_what_the_emitter_refuses(location, mode): + contract = _contract(location, mode=mode) + try: + _emit(contract) + refused = False + except SovereigntyViolationError: + refused = True + assert bool(_hook(contract)) is refused + + +def test_the_plan_hook_gives_no_verdict_without_a_policy(): + assert _hook(_contract(dict(_BQ), sovereignty=False)) is None + + +def test_plan_check_sovereignty_reports_the_gcp_hook(workspace, capsys): + contract = _multi_region_contract("EU") + path = _write(workspace, contract) + rc = _cli("plan", str(path), "--out", str(workspace / "plan.json"), "--check-sovereignty") + out = capsys.readouterr().out + assert rc == 0 + assert "Sovereignty check: PASS — source: gcp provider hook" in out From b3219988934cbf07a479a3d343b01b2c1f10ec41 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 15:47:58 +0200 Subject: [PATCH 04/10] fix(iac): the state-key migration attributes a state by its resources' clouds, not by OpenTofu's built-in provider A gcp product whose table's partitions expire keeps a terraform_data beside the table (the trigger that replaces it when its partitioning changes), under provider["terraform.io/builtin/terraform"]. state_sources counted builtin/terraform as a provider no plugin emits, so classify called every such legacy state ambiguous and refused the move, and with it every apply, dry-run and diff of the product (measured on the integration with the governance branch). The built-in provider ships inside OpenTofu and names no cloud; it is now left out, and a state of nothing but built-in resources is still not guessed. --- fluid_build/iac/state_migration.py | 28 ++++++++++++- tests/iac/test_state_migration_safety.py | 52 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/fluid_build/iac/state_migration.py b/fluid_build/iac/state_migration.py index f0d58a8e..6ebed551 100644 --- a/fluid_build/iac/state_migration.py +++ b/fluid_build/iac/state_migration.py @@ -68,6 +68,10 @@ an operator decides. Moving the wrong state is the one mistake that cannot be undone by the next apply. +OpenTofu's built-in provider (``terraform.io/builtin/terraform``, which +``terraform_data`` belongs to) is left out of the attribution: it names no +cloud, so a state is attributed by its other resources. + A read-only caller (``fluid diff``, ``fluid verify --state-drift``) asks with ``migrate=False`` and reads the old key while the move is pending, so a drift gate that runs before the first upgraded apply still sees the real state. @@ -81,7 +85,7 @@ import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, FrozenSet, Iterable, Mapping, Optional, Set, Tuple +from typing import Any, Dict, FrozenSet, Iterable, List, Mapping, Optional, Set, Tuple from . import runner from .backend import backend_location @@ -177,7 +181,17 @@ def plugin_sources(provider: str) -> FrozenSet[str]: def state_sources(resources: Iterable[Mapping[str, Any]]) -> FrozenSet[str]: - """``namespace/type`` of every provider the state's resources name.""" + """``namespace/type`` of every provider the state's resources name. + + OpenTofu's built-in provider (``terraform.io/builtin/terraform``: the + ``terraform_data`` resource, the ``terraform_remote_state`` data source) + is not one: it ships inside the binary, any module can use it, and it + says nothing about which cloud a state is. The GCP plugin writes a + ``terraform_data`` beside a partitioned table (the trigger that replaces + it when its partitioning changes), and counted as a provider no plugin + emits it made every such gcp state "ambiguous", refusing the move and + with it every apply, dry-run and diff on the product. + """ found = set() for resource in resources: match = _PROVIDER_ADDR_RE.search(str(resource.get("provider") or "")) @@ -187,17 +201,27 @@ def state_sources(resources: Iterable[Mapping[str, Any]]) -> FrozenSet[str]: found.add("") continue parts = match.group(1).lower().split("/") + if _is_builtin(parts): + continue found.add("/".join(parts[-2:])) return frozenset(found) +def _is_builtin(parts: List[str]) -> bool: + """``terraform.io/builtin/``: a provider compiled into OpenTofu itself.""" + return len(parts) >= 3 and parts[-3] == "terraform.io" and parts[-2] == "builtin" + + def classify(resources: Iterable[Mapping[str, Any]], provider: str) -> Tuple[str, str]: """``(verdict, detail)``: ``"mine"``, ``"other"`` or ``"ambiguous"``. ``detail`` names the owner for ``"other"`` and the reason for ``"ambiguous"``. See the module docstring for the rule. """ + resources = list(resources) sources = state_sources(resources) + if resources and not sources: + return "ambiguous", "only OpenTofu built-in resources, which name no cloud" by_plugin = _sources_by_plugin(provider) known = frozenset().union(*by_plugin.values()) unknown = sources - known diff --git a/tests/iac/test_state_migration_safety.py b/tests/iac/test_state_migration_safety.py index 2cedd145..3cb907fd 100644 --- a/tests/iac/test_state_migration_safety.py +++ b/tests/iac/test_state_migration_safety.py @@ -518,3 +518,55 @@ def test_a_real_apply_asks_for_the_move(monkeypatch, tmp_path): seen = _engine_until_the_guard(monkeypatch, tmp_path, dry_run=False) assert seen["migrate"] is True assert len(seen["inits"]) == 1 + + +# ── OpenTofu's built-in provider names no cloud ────────────────────────── +# +# The GCP plugin writes a ``terraform_data`` beside a table whose partitions +# expire (``lifecycle.retention`` with ``expire: true``): the trigger that +# replaces the table when its partitioning changes. OpenTofu records it under +# ``provider["terraform.io/builtin/terraform"]``. Read as a provider no plugin +# emits, it made the legacy state of every such gcp product "ambiguous", and +# the refusal blocked apply, dry-run and diff (measured on the integration of +# the governance branch with this one). + +_BUILTIN = 'provider["terraform.io/builtin/terraform"]' +_GCP_LEGACY_WITH_TRIGGER = _state( + "55555555-eeee", + _resource("google_bigquery_dataset", "hunt_retention", _GOOGLE), + _resource("google_bigquery_table", "hunt_retention_events", _GOOGLE), + _resource("terraform_data", "hunt_retention_events_partitioning", _BUILTIN), +) + + +def test_a_gcp_state_holding_the_partition_trigger_is_the_gcp_apply_s(): + resources = _GCP_LEGACY_WITH_TRIGGER["resources"] + assert mig.classify(resources, "gcp") == ("mine", "gcp") + assert mig.classify(resources, "aws")[0] == "other" + assert mig.other_clouds(resources, "aws") == frozenset({"gcp"}) + + +def test_a_state_of_only_built_in_resources_is_still_not_guessed(): + verdict, detail = mig.classify([_resource("terraform_data", "t", _BUILTIN)], "gcp") + assert verdict == "ambiguous" + assert "built-in" in detail + + +def test_the_gcp_state_with_the_trigger_is_moved_and_nothing_pins_the_built_in(fake, tmp_path): + moved = _state("66666666-ffff", *_GCP_LEGACY_WITH_TRIGGER["resources"]) + tofu = fake([_EMPTY, _EMPTY, moved], legacy=_GCP_LEGACY_WITH_TRIGGER) + workdir = tmp_path / "workdir" + workdir.mkdir() + outcome = mig.reconcile_state_key( + workdir=workdir, + current={"s3": {"bucket": "b", "key": "fluid/p/gcp/terraform.tfstate"}}, + legacy=_LEGACY, + provider="gcp", + env={}, + migrate=True, + ) + assert outcome.outcome == mig.MIGRATED + assert outcome.resources == 3 + assert len(tofu.copies) == 1 + pinned = tofu.modules["move-copy"]["terraform"].get("required_providers") or {} + assert [spec["source"] for spec in pinned.values()] == ["hashicorp/google"] From 093e209518b7bf135d321d68f73ad7741dff2bad Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 15:51:12 +0200 Subject: [PATCH 05/10] fix(apply): the Command Center run of a build-augmented apply says what each build did, and why the run failed fluid apply --mode amend-and-build reported only the infrastructure: the PATCH carried planned and applied changes and resources in phase apply, so a silver run that loaded 28 rows said nothing of it, and a bronze run whose load failed was closed failed with no error event and no message. The builds return an exit code, not an exception, and run_builds_from_args never touched the run report. run_builds_from_args now records each build on the open report: its id, its status (succeeded, failed, skipped) and, from the run record it wrote, the run id, the table facets.bigquery_load names, the landed destinations and the rows. The phase is build, and a failed build phase names its reason: build_failed:, build_not_found: or builds_all_skipped. Nothing is read when no apply report is open. --- fluid_build/build_runners/base.py | 74 +++++++++++ fluid_build/cli/_apply_cc_report.py | 73 +++++++++- .../test_apply_reports_to_command_center.py | 125 ++++++++++++++++++ 3 files changed, 270 insertions(+), 2 deletions(-) diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index ad576219..09ff7a7c 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -723,6 +723,64 @@ def _run_env(args: argparse.Namespace, plan_data: Optional[Dict[str, Any]] = Non return None +def _apply_report() -> Any: + """The running ``fluid apply``'s Command Center report, or ``None``.""" + try: + from fluid_build.cli._apply_cc_report import current_report + + return current_report() + except Exception: # noqa: BLE001 - reporting never fails a build + return None + + +def _runs_dir(contract_dir: Path, product_id: str, build_id: str) -> Optional[Path]: + """Where a build's run records are (``FileStateStore``), for ids the store accepts.""" + from ._ids import IdentifierViolation, validate_identifier + + try: + validate_identifier(product_id, kind="contract.id") + validate_identifier(build_id, kind="build.id") + except IdentifierViolation: + return None + return contract_dir / ".fluid" / "runs" / product_id / build_id / "runs" + + +def _run_ids(contract_dir: Path, product_id: str, build_id: str) -> Set[str]: + """The run ids a build has recorded so far.""" + runs = _runs_dir(contract_dir, product_id, build_id) + try: + return {p.stem for p in runs.glob("*.json")} if runs and runs.is_dir() else set() + except OSError: + return set() + + +def _report_build( + report: Any, + contract_dir: Path, + product_id: str, + build_id: str, + result: int, + runs_before: Optional[Set[str]], +) -> None: + """Record one build on ``report``, with the newest run record it wrote, if any. + + Run ids sort by time (``generate_run_id``), so the newest id this build + added is its run. Nothing is read when no ``fluid apply`` report is open. + """ + if report is None: + return + run: Optional[Dict[str, Any]] = None + runs = _runs_dir(contract_dir, product_id, build_id) + added = sorted(_run_ids(contract_dir, product_id, build_id) - (runs_before or set())) + if runs is not None and added: + try: + loaded = json.loads((runs / f"{added[-1]}.json").read_text(encoding="utf-8")) + run = loaded if isinstance(loaded, dict) else None + except (OSError, ValueError): + run = None + report.record_build(build_id=build_id, status="succeeded" if result == 0 else "failed", run=run) + + def run_builds_from_args( args: argparse.Namespace, logger: logging.Logger, @@ -909,11 +967,18 @@ def run_builds_from_args( if _b.get("id"): validate_identifier(_b["id"], kind="build.id") + # The ``fluid apply`` run report this build phase belongs to, if any + # (``cli/_apply_cc_report.py``): each build is recorded on it. + report = _apply_report() + product_id = str(contract.get("id") or "") + # Filter builds if specific ID requested if args.build_id: builds = [b for b in builds if b.get("id") == args.build_id] if not builds: LOG.error(f"Build not found: {args.build_id}") + if report is not None: + report.build_failed(f"build_not_found:{args.build_id}") return 1 # The overlay env the contract above was loaded with, decided once and @@ -936,6 +1001,7 @@ def run_builds_from_args( for build in builds: build_id = build.get("id", "unknown") + runs_before = _run_ids(contract_path.parent, product_id, build_id) if report else None if is_acquisition_build(build): sample_rows = getattr(args, "sample_rows", None) @@ -946,6 +1012,7 @@ def run_builds_from_args( dry_run=args.dry_run, sample_rows=sample_rows, ) + _report_build(report, contract_path.parent, product_id, build_id, result, runs_before) if result == 0: total_executed += 1 else: @@ -987,6 +1054,8 @@ def run_builds_from_args( expected = (contract_path.parent / repository / "dbt_project.yml").resolve() cprint(f"\n⚠️ Build '{build_id}' - dbt project not found: {expected}") total_skipped += 1 + if report is not None: + report.record_build(build_id=build_id, status="skipped") continue result = execute_dbt_build( @@ -1019,6 +1088,8 @@ def run_builds_from_args( "For Python builds, create the script at the expected path above." ) total_skipped += 1 + if report is not None: + report.record_build(build_id=build_id, status="skipped") continue # Execute build @@ -1033,6 +1104,7 @@ def run_builds_from_args( force_run=force_run, ) + _report_build(report, contract_path.parent, product_id, build_id, result, runs_before) if result == 0: total_executed += 1 else: @@ -1067,6 +1139,8 @@ def run_builds_from_args( total_skipped, ) return 0 + if report is not None: + report.build_failed("builds_all_skipped") console_error( f"Every build was skipped ({total_skipped}/{len(builds)}) — nothing " "was transformed and no rows were produced. Fix the missing build " diff --git a/fluid_build/cli/_apply_cc_report.py b/fluid_build/cli/_apply_cc_report.py index ff60f1d0..0c2a15f2 100644 --- a/fluid_build/cli/_apply_cc_report.py +++ b/fluid_build/cli/_apply_cc_report.py @@ -39,7 +39,12 @@ the contract hash the Command Center keys contract versions by, the ``--env`` it was applied for, the provider, the apply mode, the state location, the planned and applied change counts and the address of every -resource the module declares, and the timings. **Never a secret**: no header, +resource the module declares, and the timings. A build-augmented apply +(``--mode amend-and-build`` / ``replace-and-build``) adds each build it ran: +its id, how it ended, and, from the run record the build wrote, the run id, +the table it loaded and the rows it landed (``facets.bigquery_load``, +``facets.landed``, ``records_total``). Its phase is ``build``, and a failed +build names itself in the error event (``build_failed:``). **Never a secret**: no header, no environment value, no tofu output (its text can carry attribute values); a failure is reported by its typed event name and exit code only. @@ -108,6 +113,8 @@ def __init__(self, args: Any, logger: logging.Logger) -> None: self._disabled_reason: Optional[str] = None self._began = False self._finished = False + #: Why the build phase failed (``build_failed:``), when it did. + self._build_event: Optional[str] = None # -- filled by the apply engine ------------------------------------ @@ -177,6 +184,32 @@ def record_infra( } ) + def record_build( + self, + *, + build_id: str, + status: str, + run: Optional[Mapping[str, Any]] = None, + ) -> None: + """One build of a build-augmented apply: ``succeeded``, ``failed`` or ``skipped``. + + ``run`` is the run record the build wrote, if it wrote one; only its + id, the table it loaded, where it landed and the rows are kept. + """ + try: + entry: Dict[str, Any] = {"build_id": str(build_id), "status": str(status)} + entry.update(_build_facts(run)) + self.result.setdefault("builds", []).append(entry) + if status == "failed": + self.build_failed(f"build_failed:{build_id}") + except Exception as exc: # noqa: BLE001 - reporting never fails an apply + _LOG.debug("command center report: record_build failed: %s", type(exc).__name__) + + def build_failed(self, event: str) -> None: + """The build phase failed for ``event``; the first reason is the one reported.""" + if not self._build_event: + self._build_event = str(event) + # -- the run's end ------------------------------------------------- def finish(self, status: str, *, event: Optional[str] = None, exit_code: int = 0) -> None: @@ -203,13 +236,22 @@ def finish(self, status: str, *, event: Optional[str] = None, exit_code: int = 0 "duration_seconds": duration, } result = dict(self.result, exit_code=int(exit_code), **timings) + if not event and status != "success": + # A build that failed returns an exit code, not an exception. + event = self._build_event if event: result["error_event"] = str(event) + if self.result.get("dry_run"): + phase = "plan" + elif self.result.get("builds") or self._build_event: + phase = "build" + else: + phase = "apply" reporter.update_execution( self.execution_id, status=status, progress=100.0, - current_phase="plan" if self.result.get("dry_run") else "apply", + current_phase=phase, error_message=(f"fluid apply failed: {event}" if event else None), result=result, ) @@ -285,6 +327,33 @@ def _say(line: str) -> None: pass +def _build_facts(run: Optional[Mapping[str, Any]]) -> Dict[str, Any]: + """The run id, the loaded table, the landed destinations and rows of a run record. + + Nothing else from the record: its facets can carry engine output. + """ + if not isinstance(run, Mapping): + return {} + facts: Dict[str, Any] = {} + if run.get("run_id"): + facts["run_id"] = str(run["run_id"]) + facets = run.get("facets") if isinstance(run.get("facets"), Mapping) else {} + load = facets.get("bigquery_load") + if isinstance(load, Mapping): + if load.get("table"): + facts["table"] = str(load["table"]) + if isinstance(load.get("rows"), int): + facts["rows"] = int(load["rows"]) + landed = facets.get("landed") + destinations = landed.get("destinations") if isinstance(landed, Mapping) else None + if isinstance(destinations, Mapping) and destinations: + facts["destinations"] = {str(k): str(v) for k, v in destinations.items()} + succeeded = str(run.get("state") or "").lower() == "succeeded" + if "rows" not in facts and succeeded and isinstance(run.get("records_total"), int): + facts["rows"] = int(run["records_total"]) + return facts + + def _contract_facts(contract: Mapping[str, Any]) -> Dict[str, Any]: """Identity facts only: nothing from the contract's body beyond its id and versions.""" facts: Dict[str, Any] = { diff --git a/tests/cli/test_apply_reports_to_command_center.py b/tests/cli/test_apply_reports_to_command_center.py index 633d3805..96659967 100644 --- a/tests/cli/test_apply_reports_to_command_center.py +++ b/tests/cli/test_apply_reports_to_command_center.py @@ -472,3 +472,128 @@ def test_a_provider_that_cannot_be_resolved_is_reported_with_its_product( assert "platform" not in metadata assert "contract_hash" not in metadata assert post["body"]["environment"] == "gcp" + + +# ── A build-augmented apply reports its builds ──────────────────────────── +# +# Measured against a stub Command Center on the integration branch: `fluid +# apply --mode amend-and-build` on a silver product whose build loaded 28 rows +# into BigQuery was closed with an infra-only result (planned and applied +# changes, resources) in phase "apply", and a bronze run whose load failed was +# closed "failed" with no error event and no error message. + +_BUILD_ID = "summarise_subscriptions" +_LOADED_TABLE = "northwind-demo.demo_bronze.customer_subscriptions" + + +def _with_build(product: Path) -> Path: + doc = yaml.safe_load(product.read_text(encoding="utf-8")) + doc["builds"] = [ + { + "id": _BUILD_ID, + "pattern": "embedded-logic", + "engine": "sql", + "properties": {"sql": "SELECT 1 AS subscription_id"}, + } + ] + product.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") + return product + + +@pytest.fixture +def build(monkeypatch) -> Dict[str, Any]: + """The embedded-SQL build, answered: it writes the run record a BigQuery load writes.""" + from fluid_build.build_runners import base + + behaviour: Dict[str, Any] = {"rc": 0, "calls": 0} + + def _execute(build, contract, contract_dir, **_kwargs): + behaviour["calls"] += 1 + runs = Path(contract_dir) / ".fluid" / "runs" / contract["id"] / build["id"] / "runs" + runs.mkdir(parents=True, exist_ok=True) + ok = behaviour["rc"] == 0 + record: Dict[str, Any] = { + "run_id": f"01RUN{behaviour['calls']:08d}", + "state": "succeeded" if ok else "failed", + "records_total": 28 if ok else 0, + "facets": {"engine": "duckdb", "pattern": "embedded-logic"}, + } + if ok: + record["facets"]["bigquery_load"] = {"table": _LOADED_TABLE, "rows": 28} + record["facets"]["landed"] = { + "mode": "full_refresh", + "rows_from": "write", + "destinations": {"subscriptions": f"bigquery://{_LOADED_TABLE}"}, + } + (runs / f"{record['run_id']}.json").write_text(json.dumps(record), encoding="utf-8") + return behaviour["rc"] + + monkeypatch.setattr(base, "_execute_embedded_sql_build", _execute) + return behaviour + + +def test_a_build_augmented_apply_reports_each_build_and_what_it_landed( + product, tofu, build, cc, monkeypatch +): + url, recorder = cc + _configure(monkeypatch, url) + _with_build(product) + + assert _apply(product, "--mode", "amend-and-build") == 0 + assert build["calls"] == 1 + + (post,) = _by(recorder, "POST") + (patch,) = _by(recorder, "PATCH") + assert post["body"]["metadata"]["mode"] == "amend-and-build" + update = patch["body"] + assert update["status"] == "success" + assert update["current_phase"] == "build" + assert update.get("error_message") is None + result = update["result"] + assert result["applied_changes"] == {"add": 2, "change": 0, "remove": 0} + assert result["builds"] == [ + { + "build_id": _BUILD_ID, + "status": "succeeded", + "run_id": "01RUN00000001", + "table": _LOADED_TABLE, + "rows": 28, + "destinations": {"subscriptions": f"bigquery://{_LOADED_TABLE}"}, + } + ] + assert "error_event" not in result + + +def test_a_failed_build_is_reported_with_its_build_id(product, tofu, build, cc, monkeypatch): + url, recorder = cc + _configure(monkeypatch, url) + _with_build(product) + build["rc"] = 1 + + assert _apply(product, "--mode", "amend-and-build") == 1 + + (patch,) = _by(recorder, "PATCH") + update = patch["body"] + assert update["status"] == "failed" + assert update["current_phase"] == "build" + assert update["error_message"] == f"fluid apply failed: build_failed:{_BUILD_ID}" + result = update["result"] + assert result["error_event"] == f"build_failed:{_BUILD_ID}" + assert result["exit_code"] == 1 + (reported,) = result["builds"] + assert reported == {"build_id": _BUILD_ID, "status": "failed", "run_id": "01RUN00000001"} + + +def test_a_build_id_that_is_not_in_the_contract_is_the_reason( + product, tofu, build, cc, monkeypatch +): + url, recorder = cc + _configure(monkeypatch, url) + _with_build(product) + + assert _apply(product, "--mode", "amend-and-build", "--build-id", "no_such_build") == 1 + + (patch,) = _by(recorder, "PATCH") + assert patch["body"]["result"]["error_event"] == "build_not_found:no_such_build" + assert patch["body"]["current_phase"] == "build" + assert build["calls"] == 0 From 6d456fca9a311ff593fd6cf2ca35f65954bb7a80 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 15:56:14 +0200 Subject: [PATCH 06/10] fix(schedule-sync): the first sync after the env-scoped DAG layout retires the DAG it replaced An env's DAGs moved from / (dag id __) to __/ (____), and --delete-scope product mirrors only the new directory. On the demo lab the first sync after the upgrade left bronze.customer_subscriptions/ingest_subscriptions_dag.py beside bronze.customer_subscriptions__aws/, both FLUID_ENV_NAME 'aws' on the same cron, and the lab's Airflow unpauses a DAG as it parses it: two fluid apply --env aws of one product against one state at the same minute. Under --delete-scope product the file and git+ssh transports, whose destination is readable here, now retire the old directory's DAGs rendered for the same product and env with the old id, read from each file's syntax tree (never imported): an rsync --delete from an empty directory filtered to exactly those files, after the new directory synced. An env-less DAG, another env's, and any file that is not a rendered DAG stay. The other transports print the one step left to do, and the report records each superseded scope and whether its old DAGs were retired. --- fluid_build/cli/schedule_sync.py | 240 ++++++++++++++++++- tests/cli/test_schedule_sync_delete_scope.py | 234 ++++++++++++++++++ 2 files changed, 462 insertions(+), 12 deletions(-) diff --git a/fluid_build/cli/schedule_sync.py b/fluid_build/cli/schedule_sync.py index 7d1f8283..a1f9d363 100644 --- a/fluid_build/cli/schedule_sync.py +++ b/fluid_build/cli/schedule_sync.py @@ -57,6 +57,13 @@ deletes nothing. This applies to the transports that delete (file, ssh, git+ssh, s3 and gs for airflow; s3 for mwaa); az, scp, composer, astronomer, prefect and dagster never delete and ignore it. +* An env's DAGs are ``__/`` (dag ids + ``____``); forge-cli 0.16.6 and earlier wrote them to + ``/`` as ``__``. Under ``--delete-scope product`` + the first sync after the upgrade retires those old DAGs, the ones rendered + for the same product and env, where the destination can be read here (a + local path, a git+ssh clone). Every other transport prints the one step + left to do, and the report lists each case (``superseded_scopes``). CLI surface:: @@ -76,6 +83,7 @@ from __future__ import annotations import argparse +import ast import json import logging import os @@ -564,6 +572,195 @@ def _under(root: str, suffix: str) -> str: return root if not suffix else root.rstrip("/") + "/" + suffix +# ----------------------------------------------------------------------------- +# Retiring the DAGs an env's scope replaced +# ----------------------------------------------------------------------------- +# +# forge-cli 0.16.6 and earlier wrote a product's DAGs to ``/`` with +# dag_id ``__``, whatever the ``--env``. Now an env's DAGs are +# ``__/`` with dag_id ``____`` +# (``fluid_apply.schedule_scope_for`` / ``dag_id_for``). ``--delete-scope +# product`` mirrors only the new directory, so the first sync after the +# upgrade left the old DAG in place beside the new one: two DAGs applying the +# same product at the same minute (measured on the demo lab, whose Airflow +# unpauses a DAG as it parses it). The old ones are retired where this command +# can read the destination (a local path, and the clone of a git+ssh one), and +# named in a note everywhere else. + +#: The module-level names a rendered DAG file assigns (``fluid_apply.render_dag``). +_DAG_FACT_NAMES = ("PRODUCT_ID", "BUILD_ID", "FLUID_ENV_NAME") +#: A DAG file name the retirement acts on: a plain module name. +_DAG_FILE_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.\-]{0,127}\.py$") + + +def _dag_facts(path: Path) -> Optional[Dict[str, str]]: + """``PRODUCT_ID``, ``BUILD_ID``, ``FLUID_ENV_NAME`` and ``dag_id`` of a rendered DAG. + + Read from the file's syntax tree, never by importing it. ``None`` for any + file that is not one ``fluid generate`` renders. + """ + try: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 1_000_000: + return None + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError, ValueError): + return None + facts: Dict[str, str] = {} + for node in tree.body: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in _DAG_FACT_NAMES + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + facts[node.targets[0].id] = node.value.value + elif isinstance(node, ast.With): + for item in node.items: + call = item.context_expr + if not (isinstance(call, ast.Call) and getattr(call.func, "id", None) == "DAG"): + continue + for keyword in call.keywords: + value = keyword.value + if ( + keyword.arg == "dag_id" + and isinstance(value, ast.Constant) + and isinstance(value.value, str) + ): + facts["dag_id"] = value.value + return facts if set(_DAG_FACT_NAMES) | {"dag_id"} <= set(facts) else None + + +def _replaced_scopes(dags_dir: Path) -> List[Tuple[str, str, str]]: + """``(env scope, the scope it replaced, env)`` for each ``__/`` directory. + + A directory is an env's scope when every DAG in it was rendered for that + product and env, under the env's directory name and dag id. + """ + out: List[Tuple[str, str, str]] = [] + for scope in sorted(p for p in dags_dir.iterdir() if p.is_dir() and not p.is_symlink()): + dags = [_dag_facts(f) for f in sorted(scope.glob("*.py"))] + if not dags or any(d is None for d in dags): + continue + products = {d["PRODUCT_ID"] for d in dags if d} + envs = {d["FLUID_ENV_NAME"] for d in dags if d} + if len(products) != 1 or len(envs) != 1: + continue + (product,), (env,) = products, envs + if not env or scope.name != f"{product}__{env}": + continue + if all(d and d["dag_id"] == f"{product}__{env}__{d['BUILD_ID']}" for d in dags): + out.append((scope.name, product, env)) + return out + + +def _superseded_dags(directory: Path, product: str, env: str) -> List[str]: + """The files in ``directory`` that are ``product``'s DAGs for ``env`` under the old id. + + Only a DAG rendered for this product and this env, whose dag id carries no + env, qualifies: an env-less DAG (``FLUID_ENV_NAME = ''``), another env's + and any file that is not a rendered DAG stay. + """ + if directory.is_symlink() or not directory.is_dir(): + return [] + out: List[str] = [] + for path in sorted(directory.iterdir()): + if not _DAG_FILE_RE.fullmatch(path.name): + continue + facts = _dag_facts(path) + if ( + facts is not None + and facts["PRODUCT_ID"] == product + and facts["FLUID_ENV_NAME"] == env + and facts["dag_id"] == f"{product}__{facts['BUILD_ID']}" + ): + out.append(path.name) + return out + + +#: The transports whose destination this command can read, and so retire in. +_RETIRING_SCHEMES = ("file", "git+ssh") +#: The transports that mirror with deletion (``--delete-scope destination`` +#: deletes the old directory itself there). +_DELETING_SCHEMES = ("s3", "gs", "file", "ssh", "git+ssh") + + +def _report_superseded_scopes(dags_dir: Path, args: argparse.Namespace) -> List[Dict[str, Any]]: + """Each env directory that replaced an old one, and whether the old DAGs were retired. + + Where this command cannot read the destination, the one step that is left + to do is printed: without it the old DAG keeps running beside the new one. + """ + try: + replaced = _replaced_scopes(dags_dir) + except OSError: + return [] + if not replaced: + return [] + scope = _delete_scope(args) + scheme = None + if args.scheduler in ("airflow", "mwaa") and args.destination: + try: + scheme, _dest = _validate_destination(args.destination, args.scheduler) + except CLIError: + scheme = None + if args.scheduler == "mwaa": + scheme = "s3" + retired_here = ( + args.scheduler == "airflow" and scheme in _RETIRING_SCHEMES and scope == "product" + ) + mirrored = scope == "destination" and scheme in _DELETING_SCHEMES + out: List[Dict[str, Any]] = [] + for current, product, env in replaced: + handled = retired_here or mirrored + out.append({"scope": current, "replaces": product, "env": env, "old_dags_retired": handled}) + if handled: + continue + cprint( + f"[schedule-sync] note: {current}/ now holds {product}'s DAGs for env {env}. " + f"forge-cli 0.16.6 and earlier synced them to {product}/ with dag ids " + f"{product}__, and this destination cannot be read here to retire " + f"those. Delete {product}/'s DAG files for env {env} at the destination once " + f"(each is a DAG whose FLUID_ENV_NAME is {env!r}), or Airflow runs " + f"{product}__ beside {product}__{env}__.", + markup=False, + ) + return out + + +def _retire_argvs( + rsync: str, root: Path, dest_root: str, replaced: List[Tuple[str, str, str]], empty: str +) -> List[List[str]]: + """An ``rsync --delete`` from an empty directory, filtered to the superseded DAGs. + + ``root`` is where the destination can be read (a local path, or the + git+ssh clone); ``dest_root`` is how rsync names it. Only the files + :func:`_superseded_dags` names are deleted: everything else in the old + directory is excluded, and rsync never deletes an excluded file. ``-r``, + not ``-a``: the old directory keeps its own mode and times, not the + empty source's (a private temporary directory). + """ + argvs: List[List[str]] = [] + for _scope, product, env in replaced: + names = _superseded_dags(root / product, product, env) + if not names: + continue + argvs.append( + [ + rsync, + "-rv", + "--delete", + *[f"--include=/{name}" for name in names], + "--exclude=*", + "--", + empty.rstrip("/") + "/", + _under(dest_root, f"{product}/"), + ] + ) + return argvs + + def _run_units(argvs: List[List[str]], args: argparse.Namespace) -> List[Dict[str, Any]]: """Run each argv in turn; stop at the first failure.""" results: List[Dict[str, Any]] = [] @@ -873,8 +1070,16 @@ def _planned(clone_dir: str) -> List[Dict]: if clone_result["exit_code"] != 0: return results + empty = str(Path(tmp) / "empty") + Path(empty).mkdir() + retire = ( + _retire_argvs(rsync, Path(clone_dir), "./", _replaced_scopes(dags_dir), empty) + if _delete_scope(args) == "product" + else [] + ) for argv in ( *rsync_argvs, + *retire, [git, "add", "--", "."], ): result = _run_subprocess_with_cwd( @@ -1025,20 +1230,29 @@ def _planned(clone_dir: str) -> List[Dict]: # future change to _validate_destination that lets a leading-'-' # path slip through still doesn't smuggle an rsync option. root = local_dest.rstrip("/") + "/" - return _run_units( - [ + with tempfile.TemporaryDirectory(prefix="fluid-schedule-sync-empty-") as empty: + # The DAGs an env's directory replaced, once each env's own + # directory is in place (see _replaced_scopes). + retire = ( + _retire_argvs(binary, Path(root), root, _replaced_scopes(dags_dir), empty) + if _delete_scope(args) == "product" + else [] + ) + return _run_units( [ - binary, - "-av", - *(["--delete"] if delete else []), - "--", - source, - _under(root, suffix), + [ + binary, + "-av", + *(["--delete"] if delete else []), + "--", + source, + _under(root, suffix), + ] + for source, suffix, delete in units ] - for source, suffix, delete in units - ], - args, - ) + + retire, + args, + ) elif scheme == "ssh": binary = _which_or_raise("rsync") # rsync over ssh: ssh://user@host/path → user@host:/path @@ -1423,6 +1637,7 @@ def run(args, _logger: Optional[logging.Logger] = None) -> int: ) results = dispatcher(dags_dir, args) + superseded = _report_superseded_scopes(dags_dir, args) # ── Acquisition pattern: emit per-orchestrator artifacts ──────────── # When the contract carries a Bronze ``pattern: acquisition`` build, @@ -1486,6 +1701,7 @@ def run(args, _logger: Optional[logging.Logger] = None) -> int: "delete_scope": delete_scope, "dry_run": args.dry_run, "results": results, + "superseded_scopes": superseded, "overall_exit": overall_exit, } diff --git a/tests/cli/test_schedule_sync_delete_scope.py b/tests/cli/test_schedule_sync_delete_scope.py index ac243113..8d7900f5 100644 --- a/tests/cli/test_schedule_sync_delete_scope.py +++ b/tests/cli/test_schedule_sync_delete_scope.py @@ -277,3 +277,237 @@ def test_default_sync_keeps_every_other_products_dags(self, tmp_path: Path) -> N ) assert schedule_sync.run(args) == 0 assert sorted(p.name for p in root.iterdir()) == ["new.product", "orders"] + + +# ── The DAG an env's directory replaced ─────────────────────────────────── +# +# forge-cli 0.16.6 and earlier synced a product's DAGs to ``/`` with +# dag id ``__``, whatever the env. An env's DAGs now live in +# ``__/`` as ``____``, and ``--delete-scope +# product`` mirrors only that directory. Measured on the demo's bronze product: +# after the first sync on the new release the destination held both DAGs, each +# ``FLUID_ENV_NAME = 'aws'`` and ``SCHEDULE = '0 */4 * * *'``, so Airflow ran two +# ``fluid apply --env aws`` of one product against one state at the same minute. + +_PRODUCT = "bronze.customer_subscriptions" +_BUILD = "ingest_subscriptions" + + +def _dag(env: str, *, legacy: bool = False, build: str = _BUILD) -> str: + from fluid_build.schedulers.airflow import fluid_apply + + text = fluid_apply.render_dag( + product_id=_PRODUCT, + build=fluid_apply.ScheduledBuild( + build_id=build, schedule="0 */4 * * *", timezone="UTC", retries=1 + ), + env=env or None, + contract_path="contracts/customer_subscriptions/contract.fluid.yaml", + env_names=[], + ) + if legacy and env: + # What 0.16.6 rendered: the same file, with no env in its dag id. + new_id = fluid_apply.py_str_literal(f"{_PRODUCT}__{env}__{build}") + assert text.count(new_id) == 1 + text = text.replace(new_id, fluid_apply.py_str_literal(f"{_PRODUCT}__{build}")) + return text + + +def _env_artifacts(tmp_path: Path, env: str = "aws") -> Path: + """``schedule/`` as stage 3 now writes it for ``--env ``.""" + dags = tmp_path / "dist" / "artifacts" / "schedule" + scope = dags / f"{_PRODUCT}__{env}" + scope.mkdir(parents=True) + (scope / f"{_BUILD}_dag.py").write_text(_dag(env), encoding="utf-8") + return dags + + +def _upgraded_root(tmp_path: Path) -> Path: + """The lab's DAG root after a sync by 0.16.6, plus what must survive the retirement.""" + root = tmp_path / "airflow-dags" + old = root / _PRODUCT + old.mkdir(parents=True) + (old / f"{_BUILD}_dag.py").write_text(_dag("aws", legacy=True), encoding="utf-8") + (old / "removed_build_dag.py").write_text( + _dag("aws", legacy=True, build="removed_build"), encoding="utf-8" + ) + # Not this env's, not a legacy id, not a DAG: all stay. + (old / "gcp_only_dag.py").write_text( + _dag("gcp", legacy=True, build="gcp_only"), encoding="utf-8" + ) + (old / "envless_dag.py").write_text(_dag("", build="envless"), encoding="utf-8") + (old / "notes.py").write_text("# someone's helper\n", encoding="utf-8") + (root / "billing").mkdir() + (root / "billing" / "billing_dag.py").write_text("# theirs\n", encoding="utf-8") + return root + + +def _unwrapped(text: str) -> str: + """The console wraps long lines; compare without whitespace.""" + return "".join(text.split()) + + +def _dag_ids(root: Path) -> List[str]: + return sorted( + facts["dag_id"] + for path in root.rglob("*.py") + if (facts := schedule_sync._dag_facts(path)) is not None + ) + + +class TestTheDagAnEnvDirectoryReplaced: + def test_the_scope_and_the_old_dags_are_read_from_the_dag_files(self, tmp_path: Path) -> None: + dags = _env_artifacts(tmp_path) + assert schedule_sync._replaced_scopes(dags) == [(f"{_PRODUCT}__aws", _PRODUCT, "aws")] + root = _upgraded_root(tmp_path) + assert schedule_sync._superseded_dags(root / _PRODUCT, _PRODUCT, "aws") == [ + f"{_BUILD}_dag.py", + "removed_build_dag.py", + ] + # A directory of the old layout, or of anything else, replaces nothing. + assert schedule_sync._replaced_scopes(root) == [] + + def test_the_dry_run_plans_the_retirement_after_the_sync(self, tmp_path: Path) -> None: + dags = _env_artifacts(tmp_path) + root = _upgraded_root(tmp_path) + argvs = _dispatch(dags, destination=str(root)) + dest = f"{root.resolve()}/" + assert argvs[0] == [ + "/bin/rsync", + "-av", + "--delete", + "--", + f"{dags}/{_PRODUCT}__aws/", + f"{dest}{_PRODUCT}__aws/", + ] + retire = argvs[1] + assert retire[:3] == ["/bin/rsync", "-rv", "--delete"] + assert retire[3:6] == [ + f"--include=/{_BUILD}_dag.py", + "--include=/removed_build_dag.py", + "--exclude=*", + ] + assert retire[-1] == f"{dest}{_PRODUCT}/" + assert len(argvs) == 2 + assert (root / _PRODUCT / f"{_BUILD}_dag.py").exists(), "a dry run deleted" + + @pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed") + def test_the_first_sync_after_the_upgrade_retires_the_old_dag(self, tmp_path: Path) -> None: + dags = _env_artifacts(tmp_path) + root = _upgraded_root(tmp_path) + report = tmp_path / "report.json" + mode = (root / _PRODUCT).stat().st_mode + + args = _args( + dags_dir=str(dags), destination=str(root), dry_run=False, env="aws", report=str(report) + ) + assert schedule_sync.run(args) == 0 + + assert _dag_ids(root) == sorted( + [ + f"{_PRODUCT}__aws__{_BUILD}", + f"{_PRODUCT}__envless", + f"{_PRODUCT}__gcp_only", + ] + ) + assert (root / _PRODUCT / "notes.py").exists() + assert (root / "billing" / "billing_dag.py").exists() + assert (root / _PRODUCT).stat().st_mode == mode + import json + + recorded = json.loads(report.read_text(encoding="utf-8"))["superseded_scopes"] + assert recorded == [ + { + "scope": f"{_PRODUCT}__aws", + "replaces": _PRODUCT, + "env": "aws", + "old_dags_retired": True, + } + ] + + # The next sync finds nothing left to retire. + assert len(_dispatch(dags, destination=str(root))) == 1 + + def test_none_scope_retires_nothing_and_says_what_is_left( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + dags = _env_artifacts(tmp_path) + root = _upgraded_root(tmp_path) + args = _args(dags_dir=str(dags), destination=str(root), delete_scope="none") + with patch.object(schedule_sync, "_which_or_raise", side_effect=_which): + assert schedule_sync.run(args) == 0 + out = _unwrapped(capsys.readouterr().out) + assert "--include=/" not in out + assert _unwrapped(f"Delete {_PRODUCT}/'s DAG files for env aws") in out + + @pytest.mark.parametrize( + "scheduler,destination", + [ + ("airflow", "s3://b/dags/"), + ("airflow", "gs://b/dags/"), + ("airflow", "ssh://u@host/opt/dags"), + ("mwaa", "s3://mwaa/dags/"), + ], + ) + def test_a_destination_that_cannot_be_read_here_gets_the_step_to_take( + self, tmp_path: Path, scheduler: str, destination: str, capsys + ) -> None: + dags = _env_artifacts(tmp_path) + report = tmp_path / "report.json" + args = _args( + dags_dir=str(dags), scheduler=scheduler, destination=destination, report=str(report) + ) + with patch.object(schedule_sync, "_which_or_raise", side_effect=_which): + assert schedule_sync.run(args) == 0 + out = _unwrapped(capsys.readouterr().out) + assert "--include=/" not in out + assert _unwrapped(f"{_PRODUCT}__ beside {_PRODUCT}__aws__") in out + assert '"old_dags_retired": false' in report.read_text(encoding="utf-8") + + def test_mirroring_the_whole_destination_needs_no_note(self, tmp_path: Path, capsys) -> None: + dags = _env_artifacts(tmp_path) + args = _args(dags_dir=str(dags), destination="s3://b/dags/", delete_scope="destination") + with patch.object(schedule_sync, "_which_or_raise", side_effect=_which): + assert schedule_sync.run(args) == 0 + assert "note:" not in capsys.readouterr().out + + def test_a_git_ssh_destination_retires_in_its_clone_before_the_commit( + self, tmp_path: Path + ) -> None: + dags = _env_artifacts(tmp_path) + upgraded = _upgraded_root(tmp_path) + args = _args( + dags_dir=str(dags), + destination="git+ssh://git@example.com/org/dags.git", + dry_run=False, + ) + cwd_calls: List[List[str]] = [] + + def _result(argv: List[str]) -> Dict[str, Any]: + return {"argv": argv, "exit_code": 0, "stdout_tail": " M x", "stderr_tail": ""} + + def _clone(argv: List[str], **_kwargs: Any) -> Dict[str, Any]: + # The clone holds what the last sync by 0.16.6 pushed. + shutil.copytree(upgraded, argv[-1]) + return _result(argv) + + def _in_clone(argv: List[str], *, cwd: str, **_kwargs: Any) -> Dict[str, Any]: + cwd_calls.append(argv) + return _result(argv) + + with ( + patch.object(schedule_sync, "_which_or_raise", side_effect=_which), + patch.object(schedule_sync, "_run_subprocess", side_effect=_clone), + patch.object(schedule_sync, "_run_subprocess_with_cwd", side_effect=_in_clone), + ): + schedule_sync._airflow_dispatch(dags, args) + + assert [argv[:2] for argv in cwd_calls[:3]] == [ + ["/bin/rsync", "-av"], + ["/bin/rsync", "-rv"], + ["/bin/git", "add"], + ] + retire = cwd_calls[1] + assert retire[-1] == f"./{_PRODUCT}/" + assert f"--include=/{_BUILD}_dag.py" in retire + assert "--include=/envless_dag.py" not in retire From b6af57e911caab098532b29b953600bfa5973684 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 15:56:33 +0200 Subject: [PATCH 07/10] docs(changelog): the first schedule-sync after the env-scoped DAG layout retires each env's old DAG --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e79362..c6ffd875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Upgrade notes + +- **The first `fluid schedule-sync` after upgrading must retire the old DAG + of each env.** An env's Airflow DAGs now live in `__/` with + dag id `____`; 0.16.6 and earlier wrote them to + `/` as `__`. Left in place, the old DAG runs + beside the new one, and both apply the same product against the same state. + With `--delete-scope product` (the default) to a local path or a `git+ssh` + repository, the sync retires them itself: it deletes from `/` + only the DAGs rendered for the same product and env under the old id, and + keeps any other file. For every other transport it prints the step to take: + delete those DAG files at the destination once. `--delete-scope destination` + removes the old directory with the rest of what the sync does not ship. + The report's `superseded_scopes` records which case applied. + ## [0.16.6] — 2026-09-28 A Lake Formation grant that hides columns from a principal now plans. Found From fc3778779d68bad3099e72be6d48aaa659a2f186 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 16:04:56 +0200 Subject: [PATCH 08/10] refactor(apply): the current apply run lives in observability, so build_runners records builds without importing the CLI The previous commit read the run report through a function-local import of fluid_build.cli._apply_cc_report from build_runners, the reverse edge tests/observability/test_import_hygiene.py forbids. The context variable now lives in observability/apply_run.py, which build_runners already depends on; _apply_cc_report sets and reads it there. --- fluid_build/build_runners/base.py | 21 +++++------- fluid_build/cli/_apply_cc_report.py | 23 ++++++++----- fluid_build/observability/apply_run.py | 47 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 fluid_build/observability/apply_run.py diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index 09ff7a7c..b219d4ee 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -723,16 +723,6 @@ def _run_env(args: argparse.Namespace, plan_data: Optional[Dict[str, Any]] = Non return None -def _apply_report() -> Any: - """The running ``fluid apply``'s Command Center report, or ``None``.""" - try: - from fluid_build.cli._apply_cc_report import current_report - - return current_report() - except Exception: # noqa: BLE001 - reporting never fails a build - return None - - def _runs_dir(contract_dir: Path, product_id: str, build_id: str) -> Optional[Path]: """Where a build's run records are (``FileStateStore``), for ids the store accepts.""" from ._ids import IdentifierViolation, validate_identifier @@ -808,6 +798,10 @@ def run_builds_from_args( root): the contract itself, the contract a bundle's MANIFEST records, or the contract a plan records (through its bundle when it was planned from one). See :func:`fluid_build._contract_loader.source_contract_path`. + + Each build is recorded on the running ``fluid apply``'s Command Center + report, when there is one (``observability.apply_run``): its status and + the run record it wrote, and why the build phase failed. """ # Deferred imports to avoid circular import at module-load time: # base.py -> python.runner -> base.py (for _resolve_env_placeholders). @@ -967,9 +961,10 @@ def run_builds_from_args( if _b.get("id"): validate_identifier(_b["id"], kind="build.id") - # The ``fluid apply`` run report this build phase belongs to, if any - # (``cli/_apply_cc_report.py``): each build is recorded on it. - report = _apply_report() + # The running ``fluid apply``'s run report, if any: each build is recorded on it. + from fluid_build.observability.apply_run import current_apply_run + + report = current_apply_run() product_id = str(contract.get("id") or "") # Filter builds if specific ID requested diff --git a/fluid_build/cli/_apply_cc_report.py b/fluid_build/cli/_apply_cc_report.py index 0c2a15f2..325889b6 100644 --- a/fluid_build/cli/_apply_cc_report.py +++ b/fluid_build/cli/_apply_cc_report.py @@ -57,7 +57,6 @@ from __future__ import annotations import asyncio -import contextvars import functools import logging import os @@ -69,10 +68,6 @@ _LOG = logging.getLogger(__name__) -#: The report of the ``fluid apply`` running in this context, if any. -_CURRENT: contextvars.ContextVar[Optional["ApplyRunReport"]] = contextvars.ContextVar( - "fluid_apply_cc_report", default=None -) _OFF_VALUES = {"0", "false", "no", "off"} @@ -81,8 +76,13 @@ def current_report() -> Optional["ApplyRunReport"]: - """The report the running ``fluid apply`` fills, or ``None``.""" - return _CURRENT.get() + """The report the running ``fluid apply`` fills, or ``None``. + + Held in ``observability.apply_run``, where ``build_runners`` reads it too. + """ + from fluid_build.observability.apply_run import current_apply_run + + return current_apply_run() def _utc_now() -> str: @@ -511,10 +511,15 @@ def reports_apply_run(fn: Callable[..., int]) -> Callable[..., int]: @functools.wraps(fn) def wrapper(args: Any, logger: logging.Logger) -> int: + from fluid_build.observability.apply_run import ( + reset_current_apply_run, + set_current_apply_run, + ) + from ._common import CLIError report = ApplyRunReport(args, logger) - token = _CURRENT.set(report) + token = set_current_apply_run(report) try: rc = fn(args, logger) except CLIError as exc: @@ -527,6 +532,6 @@ def wrapper(args: Any, logger: logging.Logger) -> int: report.finish("success" if rc == 0 else "failed", exit_code=rc) return rc finally: - _CURRENT.reset(token) + reset_current_apply_run(token) return wrapper diff --git a/fluid_build/observability/apply_run.py b/fluid_build/observability/apply_run.py new file mode 100644 index 00000000..e5d31836 --- /dev/null +++ b/fluid_build/observability/apply_run.py @@ -0,0 +1,47 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The ``fluid apply`` run in progress in this context, for code below the CLI. + +``cli/_apply_cc_report.py`` opens one Command Center run report per ``fluid +apply`` and sets it here. ``build_runners`` records each build on it +(``record_build`` / ``build_failed``) without importing the CLI, which the +layering in ``tests/observability/test_import_hygiene.py`` forbids. +""" + +from __future__ import annotations + +import contextvars +from typing import Any, Optional + +_CURRENT: contextvars.ContextVar[Optional[Any]] = contextvars.ContextVar( + "fluid_apply_run", default=None +) + + +def current_apply_run() -> Optional[Any]: + """The report of the ``fluid apply`` running in this context, or ``None``.""" + return _CURRENT.get() + + +def set_current_apply_run(report: Optional[Any]) -> "contextvars.Token[Optional[Any]]": + """Make ``report`` the current run; pass the token to :func:`reset_current_apply_run`.""" + return _CURRENT.set(report) + + +def reset_current_apply_run(token: "contextvars.Token[Optional[Any]]") -> None: + _CURRENT.reset(token) + + +__all__ = ["current_apply_run", "reset_current_apply_run", "set_current_apply_run"] From fc9adbd76f0a67a0e6a02b45a1700542bafef708 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 20:09:23 +0200 Subject: [PATCH 09/10] docs(schedule-sync): 0.16.7 is the last release with the old DAG layout --- CHANGELOG.md | 2 +- fluid_build/cli/schedule_sync.py | 6 +++--- tests/cli/test_schedule_sync_delete_scope.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 023197bf..15ba18dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **The first `fluid schedule-sync` after upgrading must retire the old DAG of each env.** An env's Airflow DAGs now live in `__/` with - dag id `____`; 0.16.6 and earlier wrote them to + dag id `____`; 0.16.7 and earlier wrote them to `/` as `__`. Left in place, the old DAG runs beside the new one, and both apply the same product against the same state. With `--delete-scope product` (the default) to a local path or a `git+ssh` diff --git a/fluid_build/cli/schedule_sync.py b/fluid_build/cli/schedule_sync.py index a1f9d363..b318e37e 100644 --- a/fluid_build/cli/schedule_sync.py +++ b/fluid_build/cli/schedule_sync.py @@ -58,7 +58,7 @@ git+ssh, s3 and gs for airflow; s3 for mwaa); az, scp, composer, astronomer, prefect and dagster never delete and ignore it. * An env's DAGs are ``__/`` (dag ids - ``____``); forge-cli 0.16.6 and earlier wrote them to + ``____``); forge-cli 0.16.7 and earlier wrote them to ``/`` as ``__``. Under ``--delete-scope product`` the first sync after the upgrade retires those old DAGs, the ones rendered for the same product and env, where the destination can be read here (a @@ -576,7 +576,7 @@ def _under(root: str, suffix: str) -> str: # Retiring the DAGs an env's scope replaced # ----------------------------------------------------------------------------- # -# forge-cli 0.16.6 and earlier wrote a product's DAGs to ``/`` with +# forge-cli 0.16.7 and earlier wrote a product's DAGs to ``/`` with # dag_id ``__``, whatever the ``--env``. Now an env's DAGs are # ``__/`` with dag_id ``____`` # (``fluid_apply.schedule_scope_for`` / ``dag_id_for``). ``--delete-scope @@ -719,7 +719,7 @@ def _report_superseded_scopes(dags_dir: Path, args: argparse.Namespace) -> List[ continue cprint( f"[schedule-sync] note: {current}/ now holds {product}'s DAGs for env {env}. " - f"forge-cli 0.16.6 and earlier synced them to {product}/ with dag ids " + f"forge-cli 0.16.7 and earlier synced them to {product}/ with dag ids " f"{product}__, and this destination cannot be read here to retire " f"those. Delete {product}/'s DAG files for env {env} at the destination once " f"(each is a DAG whose FLUID_ENV_NAME is {env!r}), or Airflow runs " diff --git a/tests/cli/test_schedule_sync_delete_scope.py b/tests/cli/test_schedule_sync_delete_scope.py index 8d7900f5..e989c099 100644 --- a/tests/cli/test_schedule_sync_delete_scope.py +++ b/tests/cli/test_schedule_sync_delete_scope.py @@ -281,7 +281,7 @@ def test_default_sync_keeps_every_other_products_dags(self, tmp_path: Path) -> N # ── The DAG an env's directory replaced ─────────────────────────────────── # -# forge-cli 0.16.6 and earlier synced a product's DAGs to ``/`` with +# forge-cli 0.16.7 and earlier synced a product's DAGs to ``/`` with # dag id ``__``, whatever the env. An env's DAGs now live in # ``__/`` as ``____``, and ``--delete-scope # product`` mirrors only that directory. Measured on the demo's bronze product: From 15d57cfa5c5062eb427b5021108f6cb7131e5214 Mon Sep 17 00:00:00 2001 From: Speculator55005 Date: Mon, 28 Sep 2026 21:04:03 +0200 Subject: [PATCH 10/10] test(cli): drop the unused Path import --- tests/cli/test_one_contract_two_clouds_pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cli/test_one_contract_two_clouds_pipeline.py b/tests/cli/test_one_contract_two_clouds_pipeline.py index b13950f5..4d379ab9 100644 --- a/tests/cli/test_one_contract_two_clouds_pipeline.py +++ b/tests/cli/test_one_contract_two_clouds_pipeline.py @@ -31,7 +31,6 @@ import argparse import logging -from pathlib import Path import pytest import yaml