diff --git a/.github/schemas/quickstart-metadata.schema.json b/.github/schemas/quickstart-metadata.schema.json new file mode 100644 index 000000000..23bccd755 --- /dev/null +++ b/.github/schemas/quickstart-metadata.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Azure/terraform/blob/master/.github/schemas/quickstart-metadata.schema.json", + "title": "Azure Terraform quickstart deployment metadata", + "type": "object", + "additionalProperties": false, + "required": [ + "$schema", + "testResult" + ], + "properties": { + "$schema": { + "type": "string", + "const": "../../.github/schemas/quickstart-metadata.schema.json" + }, + "testResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "correlationId", + "timestamp" + ], + "properties": { + "correlationId": { + "type": "string", + "format": "uuid", + "description": "Value supplied to ARM_CORRELATION_REQUEST_ID for the validated Terraform apply." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z$", + "description": "RFC 3339 UTC time ending in Z, captured immediately before the validated Terraform apply started." + }, + "terraformVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+(?:[-+][0-9A-Za-z.-]+)?$", + "description": "Terraform CLI version used for the apply." + } + } + } + } +} diff --git a/.github/scripts/test_validate_arm_correlation.py b/.github/scripts/test_validate_arm_correlation.py new file mode 100644 index 000000000..0a0f9f709 --- /dev/null +++ b/.github/scripts/test_validate_arm_correlation.py @@ -0,0 +1,151 @@ +import importlib.util +import json +import unittest.mock +import pathlib +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("validate-arm-correlation.py") +SPEC = importlib.util.spec_from_file_location("arm_validator", SCRIPT) +VALIDATOR = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(VALIDATOR) + + +class ValidateArmCorrelationTests(unittest.TestCase): + def test_build_body_uses_string_encoded_properties(self) -> None: + body = json.loads( + VALIDATOR.build_body( + "12345678-1234-1234-1234-1234567890ab", + "2026-10-06T18:00:00Z", + ) + ) + + self.assertEqual("Requests", body["db"]) + self.assertIsInstance(body["properties"], str) + properties = json.loads(body["properties"]) + self.assertEqual( + "12345678-1234-1234-1234-1234567890ab", + properties["Parameters"]["cid"], + ) + self.assertTrue(properties["Options"]["deferpartialqueryfailures"]) + + def test_primary_row_maps_columns(self) -> None: + response = { + "Tables": [ + { + "Columns": [ + {"ColumnName": "totalWrites"}, + {"ColumnName": "failedWrites"}, + ], + "Rows": [[3, 0]], + } + ] + } + + self.assertEqual( + {"totalWrites": 3, "failedWrites": 0}, + VALIDATOR.primary_row(response), + ) + + def test_partial_query_failure_is_rejected(self) -> None: + response = { + "Tables": [ + { + "Columns": [{"ColumnName": "totalWrites"}], + "Rows": [[3]], + }, + { + "Columns": [ + {"ColumnName": "Severity"}, + {"ColumnName": "StatusCode"}, + {"ColumnName": "StatusDescription"}, + ], + "Rows": [[2, 1234, "Result truncation"]], + }, + ] + } + + with self.assertRaisesRegex(RuntimeError, "partial query failure"): + VALIDATOR.primary_row(response) + + def test_informational_query_status_is_allowed(self) -> None: + response = { + "Tables": [ + { + "Columns": [{"ColumnName": "totalWrites"}], + "Rows": [[3]], + }, + { + "Columns": [ + {"ColumnName": "Severity"}, + {"ColumnName": "StatusCode"}, + {"ColumnName": "StatusDescription"}, + ], + "Rows": [[4, 0, "Query completed successfully"]], + }, + ] + } + + self.assertEqual({"totalWrites": 3}, VALIDATOR.primary_row(response)) + + @unittest.mock.patch.object(VALIDATOR.shutil, "which") + def test_azure_cli_finds_command(self, which: unittest.mock.Mock) -> None: + which.return_value = "/usr/bin/az" + + with unittest.mock.patch.object(VALIDATOR.os, "name", "posix"): + self.assertEqual(["/usr/bin/az"], VALIDATOR.azure_cli_command()) + + def test_success_requires_writes_and_no_final_failures(self) -> None: + self.assertEqual( + [], + VALIDATOR.validate_summary( + { + "totalWrites": 3, + "completedWrites": 2, + "acceptedAsyncWrites": 1, + "failedWrites": 0, + } + ), + ) + + def test_final_failed_write_is_rejected(self) -> None: + errors = VALIDATOR.validate_summary( + { + "totalWrites": 3, + "completedWrites": 1, + "acceptedAsyncWrites": 1, + "failedWrites": 1, + } + ) + + self.assertTrue(any("final failed ARM write" in error for error in errors)) + + def test_summary_comparison_detects_ingestion_changes(self) -> None: + first = { + "totalWrites": 1, + "completedWrites": 1, + "acceptedAsyncWrites": 0, + "failedWrites": 0, + } + second = { + "totalWrites": 2, + "completedWrites": 1, + "acceptedAsyncWrites": 0, + "failedWrites": 1, + } + + self.assertNotEqual(first, second) + + def test_snapshot_requires_all_cluster_queries(self) -> None: + self.assertTrue(VALIDATOR.snapshot_is_complete([{"totalWrites": 1}], [])) + self.assertFalse( + VALIDATOR.snapshot_is_complete( + [{"totalWrites": 1}], + ["https://armprodweu.westeurope.kusto.windows.net"], + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_validate_quickstart_metadata.py b/.github/scripts/test_validate_quickstart_metadata.py new file mode 100644 index 000000000..6625c81a7 --- /dev/null +++ b/.github/scripts/test_validate_quickstart_metadata.py @@ -0,0 +1,153 @@ +import importlib.util +import datetime +import json +import pathlib +import re +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("validate-quickstart-metadata.py") +SCHEMA = SCRIPT.parent.parent / "schemas" / "quickstart-metadata.schema.json" +SPEC = importlib.util.spec_from_file_location("metadata_validator", SCRIPT) +VALIDATOR = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(VALIDATOR) + + +class ValidateMetadataTests(unittest.TestCase): + def current_timestamp(self) -> str: + return ( + datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + def write_metadata(self, data: dict) -> pathlib.Path: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + path = pathlib.Path(temporary_directory.name) / "metadata.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + def test_accepts_valid_metadata(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": self.current_timestamp(), + "terraformVersion": "1.13.0", + }, + } + ) + + self.assertEqual([], VALIDATOR.validate_metadata(path)) + + def test_rejects_invalid_correlation_id(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "not-a-guid", + "timestamp": self.current_timestamp(), + }, + } + ) + + self.assertTrue( + any("dashed UUID" in error for error in VALIDATOR.validate_metadata(path)) + ) + + def test_requires_utc_timestamp(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2026-08-18T17:00:00-07:00", + }, + } + ) + + self.assertTrue( + any( + "timestamp" in error + for error in VALIDATOR.validate_metadata(path) + ) + ) + + def test_schema_matches_python_utc_timestamp_contract(self) -> None: + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + pattern = schema["properties"]["testResult"]["properties"]["timestamp"][ + "pattern" + ] + + self.assertEqual(VALIDATOR.TIMESTAMP_RE.pattern, pattern) + self.assertIsNotNone(re.fullmatch(pattern, "2026-08-18T17:00:00Z")) + self.assertIsNotNone(re.fullmatch(pattern, "2026-08-18T17:00:00.123Z")) + for invalid in ( + "2026-08-18T17:00:00-07:00", + "2026-08-18 17:00:00Z", + "20260818T170000Z", + "2026-08-18T17:00Z", + "2026-08-18T17:00:00,123Z", + ): + self.assertIsNone(re.fullmatch(pattern, invalid)) + self.assertIsNone(VALIDATOR.parse_timestamp(invalid)) + + def test_reads_test_result_value(self) -> None: + data = {"testResult": {"correlationId": "example"}} + + self.assertEqual( + "example", + VALIDATOR.test_result_value(data, "correlationId"), + ) + self.assertIsNone(VALIDATOR.test_result_value({}, "correlationId")) + + def test_rejects_stale_timestamp(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2000-01-01T00:00:00Z", + }, + } + ) + + self.assertTrue( + any("days old" in error for error in VALIDATOR.validate_metadata(path)) + ) + + def test_allows_stale_timestamp_when_freshness_is_not_required(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2000-01-01T00:00:00Z", + }, + } + ) + + self.assertEqual( + [], + VALIDATOR.validate_metadata(path, enforce_freshness=False), + ) + + def test_recognizes_terraform_json_configuration(self) -> None: + self.assertTrue("main.tf.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES)) + self.assertTrue( + "testing.auto.tfvars.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES) + ) + + def test_metadata_change_requires_metadata(self) -> None: + self.assertTrue( + VALIDATOR.should_require_metadata(False, False, True) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate-arm-correlation.py b/.github/scripts/validate-arm-correlation.py new file mode 100644 index 000000000..f8a29d6c5 --- /dev/null +++ b/.github/scripts/validate-arm-correlation.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import pathlib +import shutil +import subprocess +import sys +import time + + +CLUSTERS = ( + "https://armprodeus.eastus.kusto.windows.net", + "https://armprodweu.westeurope.kusto.windows.net", + "https://armprodsea.southeastasia.kusto.windows.net", +) +DATABASE = "Requests" +QUERY = r""" +declare query_parameters(cid:string, started:datetime); +HttpIncomingRequests +| where PreciseTimeStamp between ((started - 1h) .. (started + 24h)) +| where correlationId =~ cid +| where userAgent has "HashiCorp Terraform" + or userAgent has "terraform-provider-azurerm" + or userAgent has "terraform-provider-azapi" +| where httpStatusCode >= 100 +| where httpMethod in ("PUT", "PATCH", "POST", "DELETE") +| extend operationKey = strcat(httpMethod, "|", operationName, "|", targetUri) +| summarize arg_max(PreciseTimeStamp, *) by operationKey +| summarize + firstSeen=min(PreciseTimeStamp), + lastSeen=max(PreciseTimeStamp), + totalWrites=count(), + completedWrites=countif(httpStatusCode between (200 .. 299) and httpStatusCode != 202), + acceptedAsyncWrites=countif(httpStatusCode == 202), + failedWrites=countif(httpStatusCode >= 400), + providers=dcount(targetResourceProvider) +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate Terraform ARM writes for a workflow correlation ID." + ) + parser.add_argument("--correlation-id", required=True) + parser.add_argument("--started-at", required=True) + parser.add_argument("--attempts", type=int, default=3) + parser.add_argument("--retry-seconds", type=int, default=30) + return parser.parse_args() + + +def build_body(correlation_id: str, started_at: str) -> str: + properties = json.dumps( + { + "Options": {"deferpartialqueryfailures": True}, + "Parameters": {"cid": correlation_id, "started": started_at}, + }, + separators=(",", ":"), + ) + return json.dumps( + {"db": DATABASE, "csl": QUERY, "properties": properties}, + separators=(",", ":"), + ) + + +def partial_query_failures(response: dict) -> list[dict]: + failures = [] + for table in response.get("Tables", []): + columns = [ + column.get("ColumnName") for column in table.get("Columns", []) + ] + if "Severity" not in columns or "StatusCode" not in columns: + continue + for values in table.get("Rows", []): + row = dict(zip(columns, values)) + severity = row.get("Severity") + if isinstance(severity, int) and severity <= 2: + failures.append(row) + return failures + + +def primary_row(response: dict) -> dict | None: + failures = partial_query_failures(response) + if failures: + details = "; ".join( + str(failure.get("StatusDescription") or failure.get("StatusCode")) + for failure in failures + ) + raise RuntimeError(f"Kusto reported a partial query failure: {details}") + + tables = response.get("Tables", []) + if not tables: + return None + table = tables[0] + rows = table.get("Rows", []) + if not rows: + return None + columns = [column["ColumnName"] for column in table.get("Columns", [])] + return dict(zip(columns, rows[0])) + + +def azure_cli_command() -> list[str]: + command = shutil.which("az") + if command and os.name != "nt": + return [command] + if os.name == "nt" and command: + python = pathlib.Path(command).resolve().parent.parent / "python.exe" + if python.is_file(): + return [str(python), "-IBm", "azure.cli"] + raise FileNotFoundError("Azure CLI was not found on PATH.") + + +def query_cluster(cluster: str, body: str) -> dict | None: + result = subprocess.run( + azure_cli_command() + + [ + "rest", + "--method", + "POST", + "--url", + f"{cluster}/v1/rest/query", + "--resource", + "https://kusto.kusto.windows.net", + "--headers", + "Content-Type=application/json", + "--body", + body, + ], + check=True, + capture_output=True, + text=True, + ) + return primary_row(json.loads(result.stdout)) + + +def validate_summary(summary: dict) -> list[str]: + errors: list[str] = [] + total_writes = int(summary.get("totalWrites") or 0) + completed_writes = int(summary.get("completedWrites") or 0) + accepted_async_writes = int(summary.get("acceptedAsyncWrites") or 0) + failed_writes = int(summary.get("failedWrites") or 0) + if total_writes == 0: + errors.append("No terminal Terraform ARM write operations were found.") + if completed_writes + accepted_async_writes == 0: + errors.append("No successful or accepted Terraform ARM writes were found.") + if failed_writes != 0: + errors.append( + f"The correlated workflow has {failed_writes} final failed ARM write(s)." + ) + return errors + + +def snapshot_is_complete(summaries: list[dict], cluster_errors: list[str]) -> bool: + return bool(summaries) and not cluster_errors + + +def main() -> int: + args = parse_args() + body = build_body(args.correlation_id, args.started_at) + attempts = max(2, args.attempts) + previous_summary = None + last_cluster_errors = [] + + for attempt in range(1, attempts + 1): + summaries = [] + cluster_errors = [] + for cluster in CLUSTERS: + print(f"Querying {cluster} (attempt {attempt})...", file=sys.stderr) + try: + row = query_cluster(cluster, body) + except ( + json.JSONDecodeError, + RuntimeError, + subprocess.CalledProcessError, + ) as error: + cluster_errors.append(cluster) + print(f"WARNING: {cluster} query failed: {error}", file=sys.stderr) + continue + if row and int(row.get("totalWrites") or 0) > 0: + row["cluster"] = cluster + summaries.append(row) + + last_cluster_errors = cluster_errors + if snapshot_is_complete(summaries, cluster_errors): + summary = { + "correlationId": args.correlation_id, + "clusters": [row["cluster"] for row in summaries], + "firstSeen": min(row["firstSeen"] for row in summaries), + "lastSeen": max(row["lastSeen"] for row in summaries), + "totalWrites": sum(int(row["totalWrites"]) for row in summaries), + "completedWrites": sum( + int(row["completedWrites"]) for row in summaries + ), + "acceptedAsyncWrites": sum( + int(row["acceptedAsyncWrites"]) for row in summaries + ), + "failedWrites": sum(int(row["failedWrites"]) for row in summaries), + "providers": sum(int(row["providers"]) for row in summaries), + } + + if summary == previous_summary: + errors = validate_summary(summary) + print(json.dumps(summary, separators=(",", ":"))) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + previous_summary = summary + print( + "Telemetry snapshot changed or was first observed; " + "waiting for a stable snapshot.", + file=sys.stderr, + ) + + if attempt < attempts: + print( + f"Waiting {args.retry_seconds}s for telemetry ingestion.", + file=sys.stderr, + ) + time.sleep(args.retry_seconds) + + if last_cluster_errors: + print( + "ERROR: ARM telemetry queries failed for: " + + ", ".join(last_cluster_errors), + file=sys.stderr, + ) + elif previous_summary is not None: + print( + "ERROR: Correlated ARM telemetry did not stabilize before timeout.", + file=sys.stderr, + ) + else: + print("ERROR: No correlated Terraform ARM writes were found.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/validate-quickstart-metadata.py b/.github/scripts/validate-quickstart-metadata.py new file mode 100644 index 000000000..9c9909752 --- /dev/null +++ b/.github/scripts/validate-quickstart-metadata.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import json +import pathlib +import re +import subprocess +import sys +import uuid + + +CONFIGURATION_SUFFIXES = ( + ".tf", + ".tf.json", + ".tfvars", + ".tfvars.json", + ".tftpl", + ".pkr.hcl", +) +EXPECTED_SCHEMA = "../../.github/schemas/quickstart-metadata.schema.json" +MAX_METADATA_AGE = datetime.timedelta(days=30) +MAX_FUTURE_SKEW = datetime.timedelta(minutes=15) +TERRAFORM_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$") +TIMESTAMP_RE = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate deployment metadata for changed Terraform quickstarts." + ) + parser.add_argument( + "folders", + nargs="+", + help="Quickstart folders. A comma-delimited argument is also accepted.", + ) + parser.add_argument( + "--base", + help="Git base ref used to require a metadata update when configuration changes.", + ) + parser.add_argument( + "--require-metadata", + action="store_true", + help="Require metadata.json even when no Git base is supplied.", + ) + return parser.parse_args() + + +def expand_folders(values: list[str]) -> list[pathlib.Path]: + folders: list[pathlib.Path] = [] + for value in values: + folders.extend( + pathlib.Path(item.strip()) + for item in value.split(",") + if item.strip() + ) + return sorted(set(folders)) + + +def changed_files(base: str, folder: pathlib.Path) -> list[str]: + result = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD", "--", folder.as_posix()], + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def metadata_at_ref(base: str, path: pathlib.Path) -> dict | None: + result = subprocess.run( + ["git", "show", f"{base}:{path.as_posix()}"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + try: + data = json.loads(result.stdout) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def test_result_value(data: dict | None, field: str) -> object: + if not data: + return None + test_result = data.get("testResult") + if not isinstance(test_result, dict): + return None + return test_result.get(field) + + +def should_require_metadata( + explicit: bool, configuration_changed: bool, metadata_changed: bool +) -> bool: + return explicit or configuration_changed or metadata_changed + + +def parse_timestamp(value: object) -> datetime.datetime | None: + if not isinstance(value, str) or not TIMESTAMP_RE.fullmatch(value): + return None + try: + parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo != datetime.timezone.utc: + return None + return parsed + + +def validate_metadata( + path: pathlib.Path, *, enforce_freshness: bool = True +) -> list[str]: + errors: list[str] = [] + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"{path}: invalid JSON: {error}"] + + if not isinstance(data, dict): + return [f"{path}: root value must be an object"] + + allowed_root = {"$schema", "testResult"} + unexpected_root = sorted(set(data) - allowed_root) + if unexpected_root: + errors.append(f"{path}: unexpected fields: {', '.join(unexpected_root)}") + if data.get("$schema") != EXPECTED_SCHEMA: + errors.append(f"{path}: $schema must be '{EXPECTED_SCHEMA}'") + + test_result = data.get("testResult") + if not isinstance(test_result, dict): + errors.append(f"{path}: testResult must be an object") + return errors + + allowed_result = {"correlationId", "timestamp", "terraformVersion"} + unexpected_result = sorted(set(test_result) - allowed_result) + if unexpected_result: + errors.append( + f"{path}: unexpected testResult fields: {', '.join(unexpected_result)}" + ) + + correlation_id = test_result.get("correlationId") + try: + parsed_id = uuid.UUID(correlation_id) + if str(parsed_id) != correlation_id.lower(): + raise ValueError + except (AttributeError, TypeError, ValueError): + errors.append(f"{path}: testResult.correlationId must be a dashed UUID") + + timestamp = parse_timestamp(test_result.get("timestamp")) + if timestamp is None: + errors.append( + f"{path}: testResult.timestamp must be an RFC 3339 UTC timestamp ending in Z" + ) + elif enforce_freshness: + now = datetime.datetime.now(datetime.timezone.utc) + if timestamp < now - MAX_METADATA_AGE: + errors.append( + f"{path}: testResult.timestamp must be no more than " + f"{MAX_METADATA_AGE.days} days old" + ) + if timestamp > now + MAX_FUTURE_SKEW: + errors.append( + f"{path}: testResult.timestamp is more than " + f"{int(MAX_FUTURE_SKEW.total_seconds() / 60)} minutes in the future" + ) + + terraform_version = test_result.get("terraformVersion") + if terraform_version is not None and ( + not isinstance(terraform_version, str) + or not TERRAFORM_VERSION_RE.fullmatch(terraform_version) + ): + errors.append( + f"{path}: testResult.terraformVersion must be a semantic version" + ) + + return errors + + +def main() -> int: + args = parse_args() + folders = expand_folders(args.folders) + errors: list[str] = [] + + if not folders: + print("No changed quickstart folders.") + return 0 + + for folder in folders: + metadata_path = folder / "metadata.json" + require_metadata = args.require_metadata + metadata_changed = False + + if args.base: + changed = changed_files(args.base, folder) + configuration_changed = any( + file_name.endswith(CONFIGURATION_SUFFIXES) for file_name in changed + ) + metadata_changed = metadata_path.as_posix() in changed + require_metadata = should_require_metadata( + require_metadata, configuration_changed, metadata_changed + ) + if configuration_changed and not metadata_changed: + errors.append( + f"{folder}: Terraform configuration changed but metadata.json " + "was not updated with a fresh correlation ID" + ) + elif configuration_changed and metadata_path.is_file(): + previous = metadata_at_ref(args.base, metadata_path) + try: + current = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + current = None + previous_id = test_result_value(previous, "correlationId") + current_id = test_result_value(current, "correlationId") + if previous_id is not None and current_id == previous_id: + errors.append( + f"{folder}: testResult.correlationId must change when " + "Terraform configuration changes" + ) + previous_timestamp = test_result_value(previous, "timestamp") + current_timestamp = test_result_value(current, "timestamp") + if ( + previous_timestamp is not None + and current_timestamp == previous_timestamp + ): + errors.append( + f"{folder}: testResult.timestamp must change when " + "Terraform configuration changes" + ) + + if require_metadata and not metadata_path.is_file(): + errors.append(f"{folder}: metadata.json is required") + continue + + if metadata_path.is_file(): + errors.extend( + validate_metadata( + metadata_path, + enforce_freshness=require_metadata or metadata_changed, + ) + ) + + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + print(f"Validated deployment metadata for {len(folders)} quickstart folder(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/e2e-pr.yaml b/.github/workflows/e2e-pr.yaml new file mode 100644 index 000000000..e12ebb066 --- /dev/null +++ b/.github/workflows/e2e-pr.yaml @@ -0,0 +1,57 @@ +name: E2E Test Code Check + +on: + pull_request: + types: [opened, reopened, synchronize] + paths: + - 'test/**' + +jobs: + fork-notice: + if: github.event.pull_request.head.repo.fork + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Report fork limitation + shell: bash + run: | + echo "### E2E tests are disabled for fork pull requests." >> "$GITHUB_STEP_SUMMARY" + echo "A maintainer must reproduce test-code changes on a branch in Azure/terraform." >> "$GITHUB_STEP_SUMMARY" + + e2e-test-code: + if: github.event.pull_request.head.repo.fork == false + runs-on: ubuntu-latest + timeout-minutes: 360 + environment: + name: test + permissions: + contents: read + id-token: write + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Run E2E test suite + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + PACKER_VERSION: ${{ vars.PACKER_VERSION }} + shell: bash + run: | + set -euo pipefail + export ARM_OIDC_REQUEST_TOKEN="${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" + export ARM_OIDC_REQUEST_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}" + docker run --rm \ + -v "$(pwd):/src" \ + -w /src/test \ + --network=host \ + -e ARM_CLIENT_ID \ + -e ARM_OIDC_REQUEST_TOKEN \ + -e ARM_OIDC_REQUEST_URL \ + -e ARM_SUBSCRIPTION_ID \ + -e ARM_TENANT_ID \ + -e ARM_USE_OIDC=true \ + mcr.microsoft.com/azterraform:latest \ + sh -c "pkenv install '${PACKER_VERSION}' && go mod tidy && go test -timeout=360m -v ./e2e" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bc41597e9..e58844da5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,56 +1,288 @@ -name: E2E Test Check +name: Validate Quickstart ARM Activity + on: - pull_request: - types: ['opened', 'synchronize'] - paths: - - '.github/**' - - '.github/workflows/**' - - 'quickstart/**' - - 'test/**' - - 'test/e2e/**' -permissions: - actions: write - contents: read - id-token: write + issue_comment: + types: [created] jobs: - e2e-check: + gate: + name: Authorize validation command + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/validate') runs-on: ubuntu-latest - environment: - name: test + timeout-minutes: 12 + permissions: + actions: read + checks: write + contents: read + issues: write + pull-requests: read + + outputs: + check_id: ${{ steps.check.outputs.check_id }} + head_sha: ${{ steps.pr.outputs.head_sha }} + head_repo: ${{ steps.pr.outputs.head_repo }} + sample_path: ${{ steps.pr.outputs.sample_path }} + steps: - - name: Checking for Fork - shell: pwsh + - name: Authorize command and resolve pull request + id: pr + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + if (context.payload.comment.body.trim() !== '/validate') { + core.setFailed('The supported command is exactly /validate.'); + return; + } + + const commenter = context.payload.comment.user.login; + const { data: permission } = + await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: commenter + }); + const allowedPermissions = new Set(['admin', 'maintain', 'write']); + if (!allowedPermissions.has(permission.permission)) { + core.setFailed( + `Not authorized to run /validate: ${commenter} has ` + + `'${permission.permission}' repository permission; write, maintain, ` + + `or admin permission is required.` + ); + return; + } + + const owner = context.repo.owner; + const repo = context.repo.repo; + const pull_number = context.payload.issue.number; + const [{ data: pr }, files] = await Promise.all([ + github.rest.pulls.get({ owner, repo, pull_number }), + github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number, per_page: 100 } + ) + ]); + + const protectedPrefixes = ['.github/workflows/', '.github/scripts/']; + const protectedFiles = [ + '.github/schemas/quickstart-metadata.schema.json', + 'GNUmakefile', + 'scripts/terraform-validate.sh' + ]; + const touchedProtected = files + .flatMap(file => [file.filename, file.previous_filename]) + .filter(Boolean) + .filter(name => + protectedPrefixes.some(prefix => name.startsWith(prefix)) || + protectedFiles.includes(name) + ) + .filter((name, index, names) => names.indexOf(name) === index); + if (touchedProtected.length > 0) { + core.setFailed( + `/validate cannot run while validation-contract files are changing: ` + + `${touchedProtected.join(', ')}. Merge validation changes separately, ` + + `canary them from the default branch, then validate quickstart changes.` + ); + return; + } + + const deployFiles = files + .map(file => file.filename) + .filter(name => + name.startsWith('quickstart/') && + /\.(tf|tf\.json|tfvars|tfvars\.json|tftpl|pkr\.hcl)$/.test(name) + ); + const samplePaths = new Set( + deployFiles.map(name => name.split('/').slice(0, 2).join('/')) + ); + if (samplePaths.size !== 1) { + core.setFailed( + `/validate requires exactly one changed deployable quickstart; found ` + + `${samplePaths.size}: ${[...samplePaths].join(', ') || ''}` + ); + return; + } + + core.setOutput('head_sha', pr.head.sha); + core.setOutput('head_repo', pr.head.repo.full_name); + core.setOutput('sample_path', [...samplePaths][0]); + + - name: React to command + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + + - name: Start pull request check + id: check + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + with: + script: | + const { data } = await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'terraform-deployment-validation', + head_sha: process.env.HEAD_SHA, + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output: { + title: 'Validating contributor-supplied ARM activity evidence', + summary: 'A maintainer ran `/validate`; waiting for static checks, then querying ARM telemetry for the contributor-supplied correlation ID.' + } + }); + core.setOutput('check_id', data.id); + + - name: Wait for pre-pull-request checks + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + REPOSITORY: ${{ github.repository }} + shell: bash run: | - $isFork = "${{ github.event.pull_request.head.repo.fork }}" - if($isFork -eq "true") { - echo "### WARNING: This workflow is disabled for forked repositories. Please follow the [release branch process](https://azure.github.io/Azure-Verified-Modules/contributing/terraform/terraform-contribution-flow/#5-create-a-pull-request-to-the-upstream-repository) if end to end tests are required." >> $env:GITHUB_STEP_SUMMARY - } - - name: checkout - if: github.event.pull_request.head.repo.fork == false - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 - - name: Get changed files - if: github.event.pull_request.head.repo.fork == false - id: changed-files - uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 + set -euo pipefail + deadline=$(( $(date +%s) + 600 )) + + while (( $(date +%s) < deadline )); do + run_json=$(gh api \ + "/repos/${REPOSITORY}/actions/workflows/pr-check.yaml/runs?head_sha=${HEAD_SHA}&per_page=1" \ + --jq '.workflow_runs[0] // {}') + status=$(jq -r '.status // empty' <<< "${run_json}") + conclusion=$(jq -r '.conclusion // empty' <<< "${run_json}") + + if [[ "${status}" == "completed" ]]; then + if [[ "${conclusion}" == "success" ]]; then + echo "Pre-pull-request checks passed for ${HEAD_SHA}." + exit 0 + fi + echo "Pre-pull-request checks concluded '${conclusion}' for ${HEAD_SHA}." >&2 + exit 1 + fi + + echo "Waiting for pre-pull-request checks (status=${status:-not-started})..." + sleep 30 + done + + echo "Timed out waiting for pre-pull-request checks." >&2 + exit 1 + + validate: + name: Validate contributor ARM activity + needs: gate + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: + name: adx-readonly + permissions: + contents: read + id-token: write + + steps: + - name: Checkout trusted validator + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - dir_names: true - separator: "," - dir_names_include_files: "quickstart/*" - files: "quickstart/**" - files_ignore: "**/TestRecord.md" - dir_names_max_depth: 2 - - name: test pr - if: github.event.pull_request.head.repo.fork == false + repository: ${{ github.repository }} + ref: ${{ github.event.repository.default_branch }} + path: trusted + sparse-checkout: | + .github/scripts/validate-quickstart-metadata.py + .github/scripts/validate-arm-correlation.py + sparse-checkout-cone-mode: false + + - name: Checkout pull request sample + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ needs.gate.outputs.head_repo }} + ref: ${{ needs.gate.outputs.head_sha }} + path: pull-request + sparse-checkout: ${{ needs.gate.outputs.sample_path }} + + - name: Validate metadata env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + run: >- + python trusted/.github/scripts/validate-quickstart-metadata.py + --require-metadata + "pull-request/${SAMPLE_PATH}" + + - name: Sign in to Azure with OIDC + uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2 + with: + tenant-id: ${{ vars.AZURE_TENANT_ID }} + client-id: ${{ vars.AZURE_CLIENT_ID }} + allow-no-subscriptions: true + + - name: Query correlated ARM activity + env: + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + shell: bash run: | - echo "change files" $ALL_CHANGED_FILES - export ARM_OIDC_REQUEST_TOKEN=$ACTIONS_ID_TOKEN_REQUEST_TOKEN - export ARM_OIDC_REQUEST_URL=$ACTIONS_ID_TOKEN_REQUEST_URL - export ARM_SUBSCRIPTION_ID=${{ secrets.ARM_SUBSCRIPTION_ID }} - export ARM_TENANT_ID=${{ secrets.ARM_TENANT_ID }} - export ARM_CLIENT_ID=${{ secrets.ARM_CLIENT_ID }} - export PACKER_VERSION=${{ vars.PACKER_VERSION }} - export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" - docker run --rm -v $(pwd):/src -w /src/test --network=host -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_OIDC_REQUEST_TOKEN -e ARM_OIDC_REQUEST_URL -e ARM_USE_OIDC=true -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest sh -c "pkenv install $PACKER_VERSION && go mod tidy && go test -timeout=360m -v ./e2e" \ No newline at end of file + set -euo pipefail + + metadata="pull-request/${SAMPLE_PATH}/metadata.json" + correlation_id=$(jq -r '.testResult.correlationId' "${metadata}") + started_at=$(jq -r '.testResult.timestamp' "${metadata}") + + summary=$( + python trusted/.github/scripts/validate-arm-correlation.py \ + --correlation-id "${correlation_id}" \ + --started-at "${started_at}" + ) + + echo "Correlation ID: ${correlation_id}" + echo "Observed: $(jq -r '.firstSeen' <<< "${summary}") through $(jq -r '.lastSeen' <<< "${summary}")" + echo "Clusters: $(jq -r '.clusters | join(\", \")' <<< "${summary}")" + echo "Writes: $(jq -r '.totalWrites' <<< "${summary}"); completed: $(jq -r '.completedWrites' <<< "${summary}"); async accepted: $(jq -r '.acceptedAsyncWrites' <<< "${summary}"); failed: $(jq -r '.failedWrites' <<< "${summary}")" + + { + echo "## Terraform ARM activity validation" + echo + echo "- Sample: \`${SAMPLE_PATH}\`" + echo "- Correlation ID: \`${correlation_id}\`" + echo "- ARM regions: $(jq -r '.clusters | join(\", \")' <<< "${summary}")" + echo "- Synchronous completed writes: $(jq -r '.completedWrites' <<< "${summary}")" + echo "- Asynchronous writes accepted by ARM: $(jq -r '.acceptedAsyncWrites' <<< "${summary}")" + echo "- Final HTTP write failures: $(jq -r '.failedWrites' <<< "${summary}")" + echo "- Resource providers: $(jq -r '.providers' <<< "${summary}")" + } >> "${GITHUB_STEP_SUMMARY}" + + report: + name: Report ARM activity validation + needs: [gate, validate] + if: always() && needs.gate.outputs.check_id != '' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + steps: + - name: Complete pull request check + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + CHECK_ID: ${{ needs.gate.outputs.check_id }} + RESULT: ${{ needs.validate.result }} + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + with: + script: | + const passed = process.env.RESULT === 'success'; + await github.rest.checks.update({ + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: Number(process.env.CHECK_ID), + status: 'completed', + conclusion: passed ? 'success' : 'failure', + output: { + title: passed + ? 'Terraform ARM activity validation passed' + : 'Terraform ARM activity validation failed', + summary: passed + ? `The contributor-supplied correlation ID recorded by \`${process.env.SAMPLE_PATH}\` maps to completed or accepted Terraform ARM writes with no observed final HTTP write failures. This does not independently bind the telemetry to the pull request or applied source.` + : `ARM activity validation failed. Open the workflow run for details, correct the metadata or deployment, then run \`/validate\` again.` + } + }); diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index eb4d6a6a6..2ede53779 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -1,10 +1,9 @@ name: Pre Pull Request Check on: pull_request: - types: [ 'opened', 'synchronize' ] + types: [ 'opened', 'reopened', 'synchronize' ] paths: - '.github/**' - - '.github/workflows/**' - 'quickstart/**' jobs: @@ -12,7 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - name: Test quickstart validation scripts + run: | + python .github/scripts/test_validate_arm_correlation.py + python .github/scripts/test_validate_quickstart_metadata.py - name: Get changed files id: changed-files uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 @@ -22,10 +27,34 @@ jobs: dir_names_include_files: "quickstart/*" files: "quickstart/**" dir_names_max_depth: 2 + - name: Normalize changed quickstart folders + id: changed-folders + env: + CANDIDATES: ${{ steps.changed-files.outputs.all_modified_files }} + shell: bash + run: | + set -euo pipefail + folders=$( + tr ',' '\n' <<< "${CANDIDATES}" | + while IFS= read -r candidate; do + candidate=$(xargs <<< "${candidate}") + if [[ -d "${candidate}" && "${candidate}" =~ ^quickstart/[^/]+$ ]]; then + echo "${candidate}" + fi + done | + sort -u | + paste -sd, - + ) + echo "folders=${folders}" >> "$GITHUB_OUTPUT" + echo "Changed quickstart folders: ${folders:-}" + - name: Validate quickstart deployment metadata + env: + CHANGED_FOLDERS: ${{ steps.changed-folders.outputs.folders }} + run: | + python .github/scripts/validate-quickstart-metadata.py \ + --base "origin/${{ github.base_ref }}" \ + "${CHANGED_FOLDERS}" - name: pr-check run: | - export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" - if [ -z "${{ github.event.number }}" ]; then - CHANGED_FOLDERS=$(find ./quickstart -maxdepth 1 -mindepth 1 -type d | tr '\n' ',') - fi - docker run --rm -v $(pwd):/src -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check + export CHANGED_FOLDERS="${{ steps.changed-folders.outputs.folders }}" + docker run --rm -v "$(pwd):/src" -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index e69de29bb..6e44ec8da 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -0,0 +1,109 @@ +# Contributing a Terraform quickstart + +Quickstart validation has two parts: + +1. **Static checks:** formatting, Terraform validation, and repository checks. +2. **ARM activity:** a maintainer verifies the contributor-supplied correlation ID against ARM telemetry. + +The repository does not deploy quickstart PR code with repository-owned Azure credentials. Contributors deploy the sample; maintainers verify its ARM activity with `/validate`. + +## Before opening a PR + +For changes to `quickstart//`: + +1. Change one deployable quickstart per PR. +2. Run `terraform fmt`, `terraform init`, `terraform validate`, and `terraform plan`. +3. Set a new `ARM_CORRELATION_REQUEST_ID`. +4. Apply the saved plan. +5. Record the correlation ID and apply start time in `metadata.json`. +6. Remove test resources after validation unless the sample says otherwise. + +AzureRM and AzAPI send `ARM_CORRELATION_REQUEST_ID` as `x-ms-correlation-request-id`. Use a new ID for each apply. + +## Deploy and record the result + +Run from the quickstart folder after authenticating to your test subscription. + +### Bash + +```bash +export ARM_CORRELATION_REQUEST_ID="$(python -c 'import uuid; print(uuid.uuid4())')" +export VALIDATION_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +terraform init +terraform validate +terraform plan -out=tfplan +terraform apply tfplan + +printf 'correlationId=%s\ntimestamp=%s\n' \ + "$ARM_CORRELATION_REQUEST_ID" \ + "$VALIDATION_STARTED_AT" +``` + +### PowerShell + +```powershell +$env:ARM_CORRELATION_REQUEST_ID = [guid]::NewGuid().ToString() +$validationStartedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + +terraform init +terraform validate +terraform plan -out=tfplan +terraform apply tfplan + +"correlationId=$env:ARM_CORRELATION_REQUEST_ID" +"timestamp=$validationStartedAt" +``` + +Use a saved plan so the reviewed plan and applied configuration are the same local operation. Never commit `tfplan`, Terraform state, credentials, subscription IDs, tenant IDs, or other environment-specific values. + +Create `metadata.json` beside the Terraform files: + +```json +{ + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2026-08-18T17:00:00Z", + "terraformVersion": "1.13.0" + } +} +``` + +`correlationId` and `timestamp` are required. The timestamp must be UTC, end in `Z`, and be no more than 30 days old. `terraformVersion` is optional. + +If a `.tf`, `.tf.json`, `.tfvars`, `.tfvars.json`, `.tftpl`, or `.pkr.hcl` file changes, deploy again and replace the correlation ID and timestamp. + +## What happens on the PR + +The automatic **Pre Pull Request Check**: + +- detects changed quickstarts; +- requires fresh metadata for deployable changes; +- validates the metadata; +- runs the existing `pr-check` target. + +After it passes, a maintainer comments: + +```text +/validate +``` + +`/validate` runs trusted code from the default branch, reads the changed quickstart's metadata, and queries the approved regional ARMProd datasets. It passes when the supplied correlation ID has at least one completed or accepted Terraform ARM write and no observed final HTTP write failures. + +The result appears as `terraform-deployment-validation` on the exact PR commit. + +### Limits + +- The correlation ID is contributor-supplied. The workflow does not bind it to the PR, sample, actor, or applied source. +- A `202 Accepted` response proves ARM accepted the request, not that provisioning eventually succeeded. +- ARM activity does not prove `terraform apply` exited successfully. +- Reviewers must still review the code and static checks. + +PRs that modify validation workflows, validators, the metadata schema, `GNUmakefile`, or `scripts/terraform-validate.sh` cannot use `/validate`. Merge and canary validation changes separately. + +Changes under `test/**` use the separate **E2E Test Code Check**. It runs only for branches in `Azure/terraform`; fork PRs do not receive repository OIDC credentials. + +## Maintainer setup + +The `adx-readonly` environment provides `AZURE_TENANT_ID` and `AZURE_CLIENT_ID`. Its subscriptionless OIDC application has read-only access to the regional ARMProd `Requests` databases and no Azure subscription role. \ No newline at end of file