diff --git a/.secrets.baseline b/.secrets.baseline index 2bfe2219..39403351 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1019,7 +1019,7 @@ "filename": "tests/iac/conftest.py", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 843 + "line_number": 844 } ], "tests/iac/test_iac_aws_apply_engine_moto.py": [ @@ -1046,7 +1046,7 @@ "filename": "tests/iac/test_iac_moto_e2e.py", "hashed_secret": "dc724af18fbdd4e59189f5fe768a5f8311527050", "is_verified": false, - "line_number": 157 + "line_number": 180 } ], "tests/integration/test_iac_cross_account_localstack.py": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-10-06T14:56:33Z" + "generated_at": "2026-10-07T14:57:38Z" } diff --git a/tests/iac/conftest.py b/tests/iac/conftest.py index 8c259ec1..f93a38f9 100644 --- a/tests/iac/conftest.py +++ b/tests/iac/conftest.py @@ -59,6 +59,7 @@ import uuid from pathlib import Path from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple +from unittest import mock import pytest @@ -763,11 +764,11 @@ def tofu_project( # (default ``http://localhost:4566``); # * the explicit opt-in flag ``FLUID_IAC_LIVE_LOCALSTACK=1`` is set. # -# Tests apply the plugin's credential-free ``main.tf.json`` with a sidecar -# ``provider.tf.json`` that aims the AWS provider at LocalStack — the same -# pattern the existing moto e2e uses, but against the heavier emulator that -# faithfully covers Athena query execution, Glue ETL jobs, and many Pro-only -# services. +# Tests compile the plugin's module for an emulator (``AWS_ENDPOINT_URL`` +# names LocalStack) and apply it with a sidecar ``provider_override.tf.json`` +# that aims each AWS service at LocalStack — the same pattern the moto e2e +# uses, but against the heavier emulator that faithfully covers Athena query +# execution, Glue ETL jobs, and many Pro-only services. # # Known LocalStack gaps (verified on the live container): # * ``redshift-serverless`` returns 501 InternalFailure → Redshift @@ -830,10 +831,10 @@ def _localstack_ready() -> tuple[bool, str]: def aws_provider_override(endpoint: str, *, region: str = "us-east-1") -> Dict[str, Any]: """A sidecar ``provider`` block aiming the AWS provider at LocalStack. - ``tofu`` merges every ``*.tf.json`` in the workdir, so this overlays - endpoint + dummy-credential config onto the plugin's credential-free - ``main.tf.json`` — emitter output stays portable and secret-free, only - the test rig knows about the emulator. + Written as an override file, ``tofu`` merges it into the provider block + the plugin emits for an emulator, overlaying each service's endpoint and + dummy credentials onto the credential-free ``main.tf.json`` — emitter + output stays portable and secret-free. """ return { "provider": { @@ -902,11 +903,24 @@ def emit( actions: Sequence[Mapping[str, Any]] = (), ) -> str: """Compile a contract through the AWS plugin + drop the LocalStack - provider override sidecar.""" + provider override sidecar. + + The module is compiled with ``AWS_ENDPOINT_URL`` naming LocalStack, as + an emulator user sets it, so the plugin emits it for an emulator. The + sidecar skips requesting the account id, and without one + ``hashicorp/aws`` 6 gives a Glue database the id ``:``, then + refuses to read it back ("unexpected format for ID (:mesh_silver), + expected catalog-id:database-name"). The emulator module names each + Glue resource's catalog (``aws._emulator_catalog_id``). It also emits + a ``provider "aws"`` block, so the sidecar is an override file, which + merges into that block, where a second plain block would be a + duplicate provider configuration. + """ plugin = get_iac_plugin("aws") - text = build_module(plugin, contract, actions=actions) + with mock.patch.dict(os.environ, {"AWS_ENDPOINT_URL": self.endpoint}): + text = build_module(plugin, contract, actions=actions) (self.workdir / "main.tf.json").write_text(text, encoding="utf-8") - (self.workdir / "provider.tf.json").write_text( + (self.workdir / "provider_override.tf.json").write_text( json.dumps(aws_provider_override(self.endpoint, region=self.region)), encoding="utf-8", ) diff --git a/tests/iac/test_iac_moto_e2e.py b/tests/iac/test_iac_moto_e2e.py index 5d96806f..6df41b33 100644 --- a/tests/iac/test_iac_moto_e2e.py +++ b/tests/iac/test_iac_moto_e2e.py @@ -23,8 +23,9 @@ patches the in-process Python ``botocore``); it needs a real HTTP endpoint. moto's ``ThreadedMotoServer`` provides exactly that — an in-process AWS API on a real ``localhost`` port, with no Docker, no -auth token, and no credentials. The emitted ``.tf.json`` is aimed at it -with a sidecar ``provider`` override. +auth token, and no credentials. ``AWS_ENDPOINT_URL`` names it, so the +plugin emits its ``.tf.json`` for an emulator, and a sidecar +``provider_override.tf.json`` aims each service at it. Skipped unless ``tofu`` is on PATH AND moto's server extra is installed (``pip install 'moto[glue,server]'`` — the ``server`` extra pulls in @@ -33,6 +34,7 @@ from __future__ import annotations +import contextlib import json from typing import Any, Dict, Iterator @@ -113,15 +115,34 @@ def tofu_env(_tofu_plugin_cache: str) -> Dict[str, str]: @pytest.fixture -def moto_endpoint() -> Iterator[str]: - """Start an in-process moto AWS server; yield its ``http://`` endpoint.""" +def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: + """Start an in-process moto AWS server; yield its ``http://`` endpoint. + + ``AWS_ENDPOINT_URL`` names it, as an emulator user sets it, so the plugin + compiles the module for an emulator. The sidecar skips requesting the + account id, and without one ``hashicorp/aws`` 6 gives a Glue database the + id ``:``, then refuses to read it back ("unexpected format for ID + (:analytics), expected catalog-id:database-name"). The emulator module + names each Glue resource's catalog (``aws._emulator_catalog_id``). + + moto's backends are process-wide: stopping a server and starting a new + one does NOT clear them, so a database one test leaves behind makes the + next test's ``CreateDatabase`` fail with ``AlreadyExistsException``. The + ``/moto-api/reset`` admin endpoint clears them, giving each test the clean + account it expects. + """ + import requests from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) server.start() try: _, port = server.get_host_and_port() - yield f"http://127.0.0.1:{port}" + endpoint = f"http://127.0.0.1:{port}" + monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) + with contextlib.suppress(Exception): + requests.post(f"{endpoint}/moto-api/reset", timeout=5) + yield endpoint finally: server.stop() @@ -129,10 +150,12 @@ def moto_endpoint() -> Iterator[str]: def _provider_override(endpoint: str) -> Dict[str, Any]: """A sidecar ``provider`` block aiming the AWS provider at moto. - ``tofu`` merges every ``*.tf.json`` in the directory, so this - overlays endpoint + dummy-credential config onto the plugin's - credential-free ``main.tf.json`` — the plugin output stays portable - and secret-free; only the test rig knows about the emulator. + Written as ``provider_override.tf.json``, an override file: ``tofu`` + merges it into the provider block the plugin emits for an emulator + (a second plain ``provider "aws"`` block would be a duplicate + configuration), overlaying each service's endpoint and dummy + credentials onto the credential-free ``main.tf.json`` — the plugin + output stays portable and secret-free. """ # Every AWS service the plugin or its test paths touch must point at # moto — adding a service later is a one-line addition to this tuple. @@ -184,7 +207,9 @@ def test_apply_destroy_cycle_against_moto( """contract -> .tf.json -> tofu init/plan/apply -> real resources -> destroy.""" # 1. Compile the contract through the AWS plugin into .tf.json. (tmp_path / "main.tf.json").write_text(build_module(get_iac_plugin("aws"), _CONTRACT)) - (tmp_path / "provider.tf.json").write_text(json.dumps(_provider_override(moto_endpoint))) + (tmp_path / "provider_override.tf.json").write_text( + json.dumps(_provider_override(moto_endpoint)) + ) workdir = str(tmp_path) env = tofu_env # Session-shared provider cache via ``TF_PLUGIN_CACHE_DIR``. @@ -254,7 +279,7 @@ def _apply_module( (workdir / "main.tf.json").write_text( build_module(plugin, contract, actions=actions), encoding="utf-8" ) - (workdir / "provider.tf.json").write_text( + (workdir / "provider_override.tf.json").write_text( json.dumps(_provider_override(moto_endpoint)), encoding="utf-8" ) init = runner.tofu_init(str(workdir), env=env)