From 7c48b733185d7561f2b03c94ef431b042deaae92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 1 Oct 2026 22:27:42 +0200 Subject: [PATCH 1/6] Harden unskip closed tests workflow Install the verified dotnet/skills workflow package at 7bdab53812ed1ce4fe2cb6b0cf84e17c8ff0f097 and add TestFX-specific exact-FQN TRX verification. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/agents/unskip-closed-tests.agent.md | 50 ++ .../test_unskip_closed_tests_verify.py | 296 ++++++++ .github/scripts/unskip_closed_tests_verify.py | 466 ++++++++++++ .github/workflows/README.md | 2 +- .../workflows/test-unskip-closed-tests.yml | 52 ++ .../workflows/unskip-closed-tests-prepare.md | 273 +++++++ .../workflows/unskip-closed-tests-shared.md | 30 + .../unskip-closed-tests-tool/ApplyEngine.cs | 709 ++++++++++++++++++ .../unskip-closed-tests-tool/ConfigLoader.cs | 206 +++++ .../Directory.Build.props | 1 + .../Directory.Build.targets | 1 + .../Directory.Packages.props | 5 + .../unskip-closed-tests-tool/GitRepository.cs | 168 +++++ .../InventoryEngine.cs | 587 +++++++++++++++ .../unskip-closed-tests-tool/IssueResolver.cs | 483 ++++++++++++ .../unskip-closed-tests-tool/JsonSupport.cs | 148 ++++ .../ManifestValidator.cs | 77 ++ .../unskip-closed-tests-tool/Models.cs | 172 +++++ .../unskip-closed-tests-tool/PathRules.cs | 72 ++ .../unskip-closed-tests-tool/Program.cs | 149 ++++ .../unskip-closed-tests-tool/TrxVerifier.cs | 117 +++ .../UnskipClosedTests.Tool.csproj | 19 + .../unskip-closed-tests-tool/global.json | 7 + .../packages.lock.json | 47 ++ .../workflows/unskip-closed-tests-verify.sh | 5 + .../workflows/unskip-closed-tests.config.json | 40 + .../workflows/unskip-closed-tests.lock.yml | 597 +++++++++------ .github/workflows/unskip-closed-tests.md | 261 ++----- 28 files changed, 4610 insertions(+), 430 deletions(-) create mode 100644 .github/agents/unskip-closed-tests.agent.md create mode 100644 .github/scripts/test_unskip_closed_tests_verify.py create mode 100644 .github/scripts/unskip_closed_tests_verify.py create mode 100644 .github/workflows/test-unskip-closed-tests.yml create mode 100644 .github/workflows/unskip-closed-tests-prepare.md create mode 100644 .github/workflows/unskip-closed-tests-shared.md create mode 100644 .github/workflows/unskip-closed-tests-tool/ApplyEngine.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/ConfigLoader.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/Directory.Build.props create mode 100644 .github/workflows/unskip-closed-tests-tool/Directory.Build.targets create mode 100644 .github/workflows/unskip-closed-tests-tool/Directory.Packages.props create mode 100644 .github/workflows/unskip-closed-tests-tool/GitRepository.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/InventoryEngine.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/IssueResolver.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/JsonSupport.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/ManifestValidator.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/Models.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/PathRules.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/Program.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/TrxVerifier.cs create mode 100644 .github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj create mode 100644 .github/workflows/unskip-closed-tests-tool/global.json create mode 100644 .github/workflows/unskip-closed-tests-tool/packages.lock.json create mode 100644 .github/workflows/unskip-closed-tests-verify.sh create mode 100644 .github/workflows/unskip-closed-tests.config.json diff --git a/.github/agents/unskip-closed-tests.agent.md b/.github/agents/unskip-closed-tests.agent.md new file mode 100644 index 0000000000..cb98481cf5 --- /dev/null +++ b/.github/agents/unskip-closed-tests.agent.md @@ -0,0 +1,50 @@ +--- +name: unskip-closed-tests +description: "Selects only source-bound, deterministically eligible .NET Ignore sites for trusted revalidation and test execution." +--- + +# Unskip Closed Tests Planner + +You are a read-only planner. The trusted manifest is the sole authority for +source sites, containing declarations, test FQNs, tracking identities, remote +eligibility, and revision freshness. + +## Required process + +1. Read `GH_AW_UNSKIP_MANIFEST` with `jq`. +2. Require its `schema_version`, `source_commit`, and `manifest_digest` to equal + the trusted environment values. +3. Consider only candidates where `decision.eligible` is `true`. +4. Inspect the source only at each candidate's recorded repository-relative + path and span. Use it to identify ambiguity, never to create a replacement + identity. +5. Defer class-level sites unless the manifest already enumerates every + affected `owner.test_fqns` entry and has no class-level deferral. +6. Select only IDs copied byte-for-byte from `candidate_id`. +7. Call exactly one allowed output and stop. + +## Mandatory deferrals + +Defer any candidate when: + +- its source path, span, owner, containing type chain, declaration identity, + test FQN, issue identity, or state appears inconsistent; +- a method's recorded owner is not its actual syntax ancestor; +- class-level inheritance, nesting, partial declarations, or incomplete test + enumeration is present; +- issue context does not clearly correspond to the ignored test even though + the deterministic remote state is eligible; +- the candidate depends on an inferred anchor, source rewrite, or remote fact. + +Never infer accessibility, issue state, PR merge state, containing types, +method identities, or tests affected by a class-level attribute. + +## Output + +For one or more selected candidates, call `apply_verified_unskips` once with +the exact manifest digest and a JSON array string of unique candidate IDs. The +safe-output job may retain fewer candidates after source, remote, build, and +TRX revalidation. + +When no candidate remains, call `noop` once. Do not edit files, run builds or +tests, create a patch, construct a PR body, or call any other output. diff --git a/.github/scripts/test_unskip_closed_tests_verify.py b/.github/scripts/test_unskip_closed_tests_verify.py new file mode 100644 index 0000000000..c36b1d258b --- /dev/null +++ b/.github/scripts/test_unskip_closed_tests_verify.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 + +import importlib.util +import os +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + + +SCRIPT_PATH = pathlib.Path(__file__).with_name("unskip_closed_tests_verify.py") +SPEC = importlib.util.spec_from_file_location("unskip_closed_tests_verify", SCRIPT_PATH) +assert SPEC is not None +assert SPEC.loader is not None +VERIFY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VERIFY) + + +class UnskipClosedTestsVerifyTests(unittest.TestCase): + def test_parse_request_requires_exact_source_and_result_identities(self) -> None: + candidate_id, source_commit, tests = VERIFY.parse_request( + { + "schema_version": "1", + "source_commit": "a" * 40, + "candidate": {"candidate_id": "candidate-1"}, + "tests": [ + { + "fqn": "Example.Tests.TestOne", + "source_path": "test/UnitTests/Example/Tests.cs", + "result_file": "results/TestOne.trx", + } + ], + } + ) + + self.assertEqual("candidate-1", candidate_id) + self.assertEqual("a" * 40, source_commit) + self.assertEqual("Example.Tests.TestOne", tests[0]["fqn"]) + self.assertEqual("test/UnitTests/Example/Tests.cs", tests[0]["source_path"]) + self.assertEqual("results/TestOne.trx", tests[0]["result_file"]) + + def test_parse_request_rejects_fabricated_or_duplicate_test_identity(self) -> None: + for fqn in ("Invented", "Example.Tests.Test One"): + with self.subTest(fqn=fqn), self.assertRaisesRegex( + VERIFY.VerificationError, "supported test identity" + ): + VERIFY.parse_request( + { + "schema_version": "1", + "source_commit": "a" * 40, + "candidate": {"candidate_id": "candidate-1"}, + "tests": [ + { + "fqn": fqn, + "source_path": "test/Example/Tests.cs", + "result_file": "results/test.trx", + } + ], + } + ) + + duplicate = { + "fqn": "Example.Tests.TestOne", + "source_path": "test/Example/Tests.cs", + "result_file": "results/test.trx", + } + with self.assertRaisesRegex(VERIFY.VerificationError, "duplicate"): + VERIFY.parse_request( + { + "schema_version": "1", + "source_commit": "a" * 40, + "candidate": {"candidate_id": "candidate-1"}, + "tests": [duplicate, duplicate], + } + ) + + def test_find_project_uses_nearest_unambiguous_project(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "test" / "UnitTests" / "Example" / "Tests.cs" + source.parent.mkdir(parents=True) + source.write_text("class Tests {}", encoding="utf-8") + project = source.parent / "Example.csproj" + project.write_text("", encoding="utf-8") + + self.assertEqual( + project, + VERIFY.find_project( + root, "test/UnitTests/Example/Tests.cs" + ), + ) + + def test_find_project_rejects_ambiguous_or_unmapped_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "test" / "Example" / "Tests.cs" + source.parent.mkdir(parents=True) + source.write_text("class Tests {}", encoding="utf-8") + (source.parent / "One.csproj").write_text("", encoding="utf-8") + (source.parent / "Two.csproj").write_text("", encoding="utf-8") + + with self.assertRaisesRegex(VERIFY.VerificationError, "ambiguous"): + VERIFY.find_project(root, "test/Example/Tests.cs") + + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "test" / "Example" / "Tests.cs" + source.parent.mkdir(parents=True) + source.write_text("class Tests {}", encoding="utf-8") + + with self.assertRaisesRegex(VERIFY.VerificationError, "No owning"): + VERIFY.find_project(root, "test/Example/Tests.cs") + + def test_select_target_framework_prefers_portable_net8(self) -> None: + self.assertEqual( + "net8.0", + VERIFY.select_target_framework(("net462", "net8.0", "net10.0")), + ) + self.assertEqual( + "net9.0", + VERIFY.select_target_framework(("net9.0", "net10.0")), + ) + with self.assertRaisesRegex(VERIFY.VerificationError, "No portable"): + VERIFY.select_target_framework(("net462", "net8.0-windows")) + + def test_repository_bootstrap_runs_pack_once_for_acceptance(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + project = ( + root + / "test" + / "IntegrationTests" + / "Example.Acceptance.IntegrationTests" + / "Example.Acceptance.IntegrationTests.csproj" + ) + project.parent.mkdir(parents=True) + project.write_text("", encoding="utf-8") + build_script = root / ("build.cmd" if os.name == "nt" else "build.sh") + build_script.write_text("", encoding="utf-8") + commands: list[list[str]] = [] + + def runner( + command: list[str], **kwargs: object + ) -> subprocess.CompletedProcess[str]: + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + with mock.patch.object( + VERIFY, "validate_revision", return_value=None + ), mock.patch.dict(os.environ, {"RUNNER_TEMP": str(root / "runner-temp")}): + VERIFY.ensure_repository_built(root, "a" * 40, True, 30, runner) + VERIFY.ensure_repository_built(root, "a" * 40, True, 30, runner) + + expected = ( + [ + [ + os.environ.get("COMSPEC", "cmd.exe"), + "/d", + "/c", + str(build_script), + "-pack", + ] + ] + if os.name == "nt" + else [[str(build_script), "-pack"]] + ) + self.assertEqual(expected, commands) + + def test_verify_tests_runs_exact_fqn_and_requested_trx_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "test" / "Example" / "Tests.cs" + source.parent.mkdir(parents=True) + source.write_text("class Tests {}", encoding="utf-8") + project = source.parent / "Example.csproj" + project.write_text("", encoding="utf-8") + result = root / "results" / "TestOne.trx" + commands: list[list[str]] = [] + + def property_runner( + command: list[str], **kwargs: object + ) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + command, + 0, + stdout=( + '{"Properties":{"TargetFrameworks":"net8.0;net10.0",' + '"TargetFramework":"","OutputType":"Exe"}}' + ), + stderr="", + ) + + def command_runner( + command: list[str], **kwargs: object + ) -> subprocess.CompletedProcess[str]: + commands.append(command) + if "--report-trx-filename" in command: + result.parent.mkdir(parents=True, exist_ok=True) + result.write_text("", encoding="utf-8") + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + with mock.patch.object( + VERIFY, "validate_revision", return_value=None + ), mock.patch.object( + VERIFY, "ensure_repository_built", return_value=None + ), mock.patch.dict(os.environ, {"RUNNER_TEMP": str(root)}): + VERIFY.verify_tests( + root, + "a" * 40, + ( + { + "fqn": "Example.Tests.TestOne", + "source_path": "test/Example/Tests.cs", + "result_file": str(result), + }, + ), + 30, + property_runner=property_runner, + command_runner=command_runner, + ) + + self.assertEqual("build", commands[0][1]) + self.assertIn("-p:EnableCodeCoverage=False", commands[0]) + self.assertIn("-bl:{}", commands[0]) + self.assertIn("--filter-uid", commands[1]) + self.assertIn("-p:EnableCodeCoverage=False", commands[1]) + self.assertIn("-bl:{}", commands[1]) + self.assertEqual( + "Example.Tests.TestOne", + commands[1][commands[1].index("--filter-uid") + 1], + ) + self.assertEqual( + result.name, + commands[1][commands[1].index("--report-trx-filename") + 1], + ) + self.assertTrue( + pathlib.Path( + commands[1][commands[1].index("--results-directory") + 1] + ).samefile(result.parent) + ) + + def test_verify_tests_rejects_missing_requested_result(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "test" / "Example" / "Tests.cs" + source.parent.mkdir(parents=True) + source.write_text("class Tests {}", encoding="utf-8") + (source.parent / "Example.csproj").write_text( + "", encoding="utf-8" + ) + + def property_runner( + command: list[str], **kwargs: object + ) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + command, + 0, + stdout=( + '{"Properties":{"TargetFrameworks":"net8.0",' + '"TargetFramework":"","OutputType":"Exe"}}' + ), + stderr="", + ) + + def command_runner( + command: list[str], **kwargs: object + ) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + with mock.patch.object( + VERIFY, "validate_revision", return_value=None + ), mock.patch.object( + VERIFY, "ensure_repository_built", return_value=None + ), mock.patch.dict(os.environ, {"RUNNER_TEMP": str(root)}): + with self.assertRaisesRegex( + VERIFY.VerificationError, "did not create requested TRX" + ): + VERIFY.verify_tests( + root, + "a" * 40, + ( + { + "fqn": "Example.Tests.TestOne", + "source_path": "test/Example/Tests.cs", + "result_file": str(root / "results" / "missing.trx"), + }, + ), + 30, + property_runner=property_runner, + command_runner=command_runner, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/unskip_closed_tests_verify.py b/.github/scripts/unskip_closed_tests_verify.py new file mode 100644 index 0000000000..486c697e8f --- /dev/null +++ b/.github/scripts/unskip_closed_tests_verify.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import pathlib +import re +import subprocess +import sys +from collections.abc import Callable, Sequence +from typing import Any + + +MAX_REQUEST_BYTES = 4 * 1024 * 1024 +FQN_PATTERN = re.compile( + r"^(?:@?[A-Za-z_][A-Za-z0-9_]*\.)+@?[A-Za-z_][A-Za-z0-9_]*$" +) +TFM_PATTERN = re.compile(r"^net(?P[0-9]+)(?:\.[0-9]+)?(?:-[A-Za-z0-9.-]+)?$") + + +class VerificationError(ValueError): + pass + + +def require_dict(value: Any, context: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise VerificationError(f"{context} must be an object") + return value + + +def require_list(value: Any, context: str) -> list[Any]: + if not isinstance(value, list): + raise VerificationError(f"{context} must be an array") + return value + + +def require_string(value: Any, context: str) -> str: + if not isinstance(value, str) or not value: + raise VerificationError(f"{context} must be a non-empty string") + return value + + +def normalize_relative_path(value: Any, context: str) -> pathlib.PurePosixPath: + text = require_string(value, context) + if "\\" in text: + raise VerificationError(f"{context} must use '/' separators") + path = pathlib.PurePosixPath(text) + if path.is_absolute() or ".." in path.parts or text.startswith("./"): + raise VerificationError(f"{context} must be repository-relative") + return path + + +def load_request(path: pathlib.Path) -> dict[str, Any]: + try: + if path.stat().st_size > MAX_REQUEST_BYTES: + raise VerificationError("Verification request is too large") + with path.open(encoding="utf-8") as stream: + return require_dict(json.load(stream), "verification request") + except VerificationError: + raise + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise VerificationError(f"Cannot read verification request: {error}") from error + + +def parse_request(value: Any) -> tuple[str, str, tuple[dict[str, str], ...]]: + request = require_dict(value, "verification request") + if request.get("schema_version") != "1": + raise VerificationError("verification request.schema_version must be '1'") + candidate = require_dict( + request.get("candidate"), "verification request.candidate" + ) + candidate_id = require_string( + candidate.get("candidate_id"), + "verification request.candidate.candidate_id", + ) + source_commit = require_string( + request.get("source_commit"), "verification request.source_commit" + ) + if not re.fullmatch(r"[0-9a-fA-F]{40,64}", source_commit): + raise VerificationError( + "verification request.source_commit must be a full object id" + ) + + tests = [] + for index, value in enumerate( + require_list(request.get("tests"), "verification request.tests") + ): + context = f"verification request.tests[{index}]" + test = require_dict(value, context) + fqn = require_string(test.get("fqn"), f"{context}.fqn") + if not FQN_PATTERN.fullmatch(fqn): + raise VerificationError(f"{context}.fqn is not a supported test identity") + source_path = normalize_relative_path( + test.get("source_path"), f"{context}.source_path" + ) + if source_path.suffix.casefold() != ".cs": + raise VerificationError(f"{context}.source_path must identify C# source") + result_file = require_string( + test.get("result_file"), f"{context}.result_file" + ) + tests.append( + { + "fqn": fqn, + "source_path": source_path.as_posix(), + "result_file": result_file, + } + ) + if not tests: + raise VerificationError("verification request.tests must not be empty") + if len({test["fqn"] for test in tests}) != len(tests): + raise VerificationError("verification request contains duplicate test identities") + return candidate_id, source_commit.lower(), tuple(tests) + + +def run_git(root: pathlib.Path, *arguments: str) -> str: + try: + completed = subprocess.run( + ["git", *arguments], + cwd=root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + ) + except subprocess.CalledProcessError as error: + stderr = error.stderr.strip() if error.stderr else str(error) + raise VerificationError(f"git {' '.join(arguments)} failed: {stderr}") from error + return completed.stdout.strip() + + +def validate_revision(root: pathlib.Path, source_commit: str) -> None: + head = run_git(root, "rev-parse", "HEAD").lower() + if head != source_commit: + raise VerificationError( + f"Repository revision changed from {source_commit} to {head}" + ) + + +def find_project(root: pathlib.Path, source_path: str) -> pathlib.Path: + source = root / pathlib.PurePosixPath(source_path) + if not source.is_file(): + raise VerificationError(f"Source file does not exist: {source_path}") + if root.resolve() not in source.resolve().parents: + raise VerificationError(f"Source file escapes repository: {source_path}") + + directory = source.parent + while directory != root.parent: + projects = sorted(directory.glob("*.csproj")) + if len(projects) == 1: + return projects[0] + if len(projects) > 1: + raise VerificationError( + f"Source project is ambiguous for {source_path}: " + + ", ".join(project.name for project in projects) + ) + if directory == root: + break + directory = directory.parent + raise VerificationError(f"No owning test project found for {source_path}") + + +def dotnet_path(root: pathlib.Path) -> str: + executable = "dotnet.exe" if os.name == "nt" else "dotnet" + local = root / ".dotnet" / executable + return str(local) if local.is_file() else "dotnet" + + +def parse_msbuild_properties(output: str) -> dict[str, str]: + start = output.find("{") + if start >= 0: + try: + value = json.loads(output[start:]) + properties = require_dict(value.get("Properties"), "MSBuild properties") + return { + str(name): str(property_value) + for name, property_value in properties.items() + } + except (json.JSONDecodeError, VerificationError, AttributeError): + pass + + properties: dict[str, str] = {} + for line in output.splitlines(): + name, separator, value = line.partition("=") + if separator and name in ( + "TargetFrameworks", + "TargetFramework", + "OutputType", + ): + properties[name] = value.strip() + if not properties: + raise VerificationError("Cannot parse evaluated test project properties") + return properties + + +def evaluate_project( + root: pathlib.Path, + project: pathlib.Path, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> tuple[str, str]: + command = [ + dotnet_path(root), + "msbuild", + str(project), + "-nologo", + "-getProperty:TargetFrameworks", + "-getProperty:TargetFramework", + "-getProperty:OutputType", + ] + try: + completed = runner( + command, + cwd=root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + ) + except subprocess.CalledProcessError as error: + stderr = error.stderr.strip() if error.stderr else str(error) + raise VerificationError( + f"Cannot evaluate {project.relative_to(root)}: {stderr}" + ) from error + properties = parse_msbuild_properties(completed.stdout) + output_type = properties.get("OutputType", "") + if output_type.casefold() != "exe": + raise VerificationError( + f"{project.relative_to(root)} is not an executable test project" + ) + frameworks = [ + framework + for framework in properties.get( + "TargetFrameworks", properties.get("TargetFramework", "") + ).split(";") + if framework + ] + target_framework = select_target_framework(frameworks) + return target_framework, output_type + + +def select_target_framework(frameworks: Sequence[str]) -> str: + if "net8.0" in frameworks: + return "net8.0" + candidates = [] + for framework in frameworks: + match = TFM_PATTERN.fullmatch(framework) + if match and "." in framework and "-" not in framework: + candidates.append((int(match.group("version")), framework)) + if not candidates: + raise VerificationError( + "No portable .NET target framework is available for verification" + ) + return min(candidates)[1] + + +def validate_result_path(root: pathlib.Path, value: str) -> pathlib.Path: + path = pathlib.Path(value) + if not path.is_absolute(): + path = root / pathlib.PurePosixPath(value) + resolved = path.resolve() + runner_temp = os.environ.get("RUNNER_TEMP") + allowed_root = pathlib.Path(runner_temp).resolve() if runner_temp else root.resolve() + if resolved != allowed_root and allowed_root not in resolved.parents: + raise VerificationError( + f"Requested result file is outside the trusted output root: {value}" + ) + if resolved.suffix.casefold() != ".trx": + raise VerificationError(f"Requested result file must use .trx: {value}") + return resolved + + +def run_checked( + command: list[str], + root: pathlib.Path, + timeout_seconds: int, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> None: + try: + runner( + command, + cwd=root, + check=True, + timeout=timeout_seconds, + stdout=sys.stdout, + stderr=sys.stderr, + text=True, + encoding="utf-8", + env={ + **os.environ, + "DOTNET_ROLL_FORWARD": os.environ.get( + "DOTNET_ROLL_FORWARD", "Major" + ), + "DOTNET_ROLL_FORWARD_TO_PRERELEASE": os.environ.get( + "DOTNET_ROLL_FORWARD_TO_PRERELEASE", "1" + ), + }, + ) + except subprocess.TimeoutExpired as error: + raise VerificationError( + f"Command timed out after {timeout_seconds} seconds: {' '.join(command)}" + ) from error + except subprocess.CalledProcessError as error: + raise VerificationError( + f"Command exited with {error.returncode}: {' '.join(command)}" + ) from error + + +def requires_packed_packages(root: pathlib.Path, project: pathlib.Path) -> bool: + relative = project.relative_to(root) + return any( + part.endswith(".Acceptance.IntegrationTests") for part in relative.parts + ) + + +def ensure_repository_built( + root: pathlib.Path, + source_commit: str, + requires_pack: bool, + timeout_seconds: int, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> None: + runner_temp = pathlib.Path(os.environ.get("RUNNER_TEMP", root / "artifacts" / "tmp")) + marker_kind = "packed" if requires_pack else "built" + marker = runner_temp / "testfx-unskip" / f"{marker_kind}-{source_commit}" + if marker.is_file(): + return + + build_script = root / ("build.cmd" if os.name == "nt" else "build.sh") + if not build_script.is_file(): + raise VerificationError(f"Repository build script is missing: {build_script}") + command = ( + [os.environ.get("COMSPEC", "cmd.exe"), "/d", "/c", str(build_script)] + if os.name == "nt" + else [str(build_script)] + ) + if requires_pack: + command.append("-pack") + run_checked( + command, + root, + timeout_seconds, + runner, + ) + validate_revision(root, source_commit) + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text(source_commit + "\n", encoding="utf-8") + if requires_pack: + built_marker = marker.parent / f"built-{source_commit}" + built_marker.write_text(source_commit + "\n", encoding="utf-8") + + +def verify_tests( + root: pathlib.Path, + source_commit: str, + tests: tuple[dict[str, str], ...], + timeout_seconds: int, + *, + property_runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + command_runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> None: + validate_revision(root, source_commit) + projects = {find_project(root, test["source_path"]) for test in tests} + if len(projects) != 1: + raise VerificationError( + "A single verification request must map to exactly one test project" + ) + project = projects.pop() + ensure_repository_built( + root, + source_commit, + requires_packed_packages(root, project), + timeout_seconds, + command_runner, + ) + target_framework, _ = evaluate_project(root, project, property_runner) + + run_checked( + [ + dotnet_path(root), + "build", + str(project), + "-c", + "Debug", + "-f", + target_framework, + "--no-restore", + "-p:EnableCodeCoverage=False", + "-bl:{}", + ], + root, + timeout_seconds, + command_runner, + ) + validate_revision(root, source_commit) + + for test in tests: + result_file = validate_result_path(root, test["result_file"]) + result_file.parent.mkdir(parents=True, exist_ok=True) + result_file.unlink(missing_ok=True) + run_checked( + [ + dotnet_path(root), + "run", + "--project", + str(project), + "-c", + "Debug", + "-f", + target_framework, + "--no-build", + "--no-restore", + "-p:EnableCodeCoverage=False", + "-bl:{}", + "--", + "--filter-uid", + test["fqn"], + "--report-trx", + "--report-trx-filename", + result_file.name, + "--results-directory", + str(result_file.parent), + ], + root, + timeout_seconds, + command_runner, + ) + if not result_file.is_file(): + raise VerificationError( + f"Test runner did not create requested TRX: {result_file}" + ) + validate_revision(root, source_commit) + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run one trusted TestFX unskip verification request." + ) + parser.add_argument("request", type=pathlib.Path) + parser.add_argument( + "--repository-root", + type=pathlib.Path, + default=pathlib.Path.cwd(), + ) + parser.add_argument("--timeout-seconds", type=int, default=900) + return parser + + +def main() -> int: + arguments = create_parser().parse_args() + try: + _, source_commit, tests = parse_request(load_request(arguments.request)) + verify_tests( + arguments.repository_root.resolve(), + source_commit, + tests, + arguments.timeout_seconds, + ) + except VerificationError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index dee60bc072..70e879b560 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -349,7 +349,7 @@ the cause. | [`resource-lock-refactoring.md`](./resource-lock-refactoring.md) | Daily + manual | Prepares one bounded test project for safe parallel execution by eliminating shared state or applying the narrowest appropriate `[ResourceLock]`, then opens a draft PR. | | [`repository-quality-improver.md`](./repository-quality-improver.md) | Weekday schedule + manual | Daily analysis of repository quality, rotating focus areas. Opens tracking issues like this one. | | [`daily-file-diet.md`](./daily-file-diet.md) | Daily + manual | Identifies oversized source files and opens actionable refactoring issues. | -| [`unskip-closed-tests.md`](./unskip-closed-tests.md) | Weekly + manual | Finds tests skipped via `[Ignore("…#issue")]` whose tracking issue is now closed, verifies they pass, and opens a PR re-enabling them. | +| [`unskip-closed-tests.md`](./unskip-closed-tests.md) | Weekly + manual | Builds a source-bound Ignore inventory at the exact commit, accepts only completed issues or merged PRs, and opens one draft PR only for tests proven executed and passed in structured TRX results. | | [`duplicate-code-detector.md`](./duplicate-code-detector.md) | Schedule + manual | Identifies duplicate code patterns and suggests refactoring opportunities. | | [`malicious-code-scan.md`](./malicious-code-scan.md) | Schedule + manual | Reviews code changes from the last 3 days for suspicious patterns indicating malicious or agentic threats. | | [`markdown-linter.md`](./markdown-linter.md) | Schedule + manual | Runs Markdown quality checks using markdownlint-cli2 and opens issues for violations. | diff --git a/.github/workflows/test-unskip-closed-tests.yml b/.github/workflows/test-unskip-closed-tests.yml new file mode 100644 index 0000000000..dbd5c7f710 --- /dev/null +++ b/.github/workflows/test-unskip-closed-tests.yml @@ -0,0 +1,52 @@ +name: Test unskip closed tests helper + +on: + pull_request: + paths: + - '.github/scripts/unskip_closed_tests_verify.py' + - '.github/scripts/test_unskip_closed_tests_verify.py' + - '.github/agents/unskip-closed-tests.agent.md' + - '.github/workflows/unskip-closed-tests.config.json' + - '.github/workflows/unskip-closed-tests-prepare.md' + - '.github/workflows/unskip-closed-tests-shared.md' + - '.github/workflows/unskip-closed-tests-tool/**' + - '.github/workflows/unskip-closed-tests-verify.sh' + - '.github/workflows/test-unskip-closed-tests.yml' + - '.github/workflows/unskip-closed-tests.md' + push: + branches: + - main + - 'rel/*' + paths: + - '.github/scripts/unskip_closed_tests_verify.py' + - '.github/scripts/test_unskip_closed_tests_verify.py' + - '.github/agents/unskip-closed-tests.agent.md' + - '.github/workflows/unskip-closed-tests.config.json' + - '.github/workflows/unskip-closed-tests-prepare.md' + - '.github/workflows/unskip-closed-tests-shared.md' + - '.github/workflows/unskip-closed-tests-tool/**' + - '.github/workflows/unskip-closed-tests-verify.sh' + - '.github/workflows/test-unskip-closed-tests.yml' + - '.github/workflows/unskip-closed-tests.md' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: test-unskip-closed-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test unskip closed tests helper + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - run: python .github/scripts/test_unskip_closed_tests_verify.py diff --git a/.github/workflows/unskip-closed-tests-prepare.md b/.github/workflows/unskip-closed-tests-prepare.md new file mode 100644 index 0000000000..ba03c38322 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-prepare.md @@ -0,0 +1,273 @@ +--- +description: >- + Deterministic source inventory, GitHub eligibility resolution, and trusted + safe-output publication for Unskip Closed Tests. + +jobs: + collect-unskip-candidates: + name: Collect verified unskip candidates + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: read + pull-requests: read + outputs: + eligible-count: ${{ steps.collect.outputs.eligible-count }} + source-commit: ${{ steps.collect.outputs.source-commit }} + manifest-digest: ${{ steps.collect.outputs.manifest-digest }} + steps: + - name: Checkout trusted source revision + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "8.0.x" + + - name: Restore trusted inventory tool + working-directory: .github/workflows/unskip-closed-tests-tool + run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode + + - name: Inventory source and resolve tracking items + id: collect + shell: bash + working-directory: .github/workflows/unskip-closed-tests-tool + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_COMMIT: ${{ github.sha }} + RAW_INVENTORY: ${{ runner.temp }}/unskip-closed-tests-inventory.json + RESOLVED_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest.json + run: | + set -euo pipefail + + set +e + dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- inventory \ + --repo-root "$GITHUB_WORKSPACE" \ + --repository "$EXPECTED_REPOSITORY" \ + --source-commit "$EXPECTED_COMMIT" \ + --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ + --output "$RAW_INVENTORY" + INVENTORY_EXIT=$? + set -e + if [ "$INVENTORY_EXIT" -ne 0 ] && [ "$INVENTORY_EXIT" -ne 10 ]; then + exit "$INVENTORY_EXIT" + fi + + set +e + dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- resolve \ + --manifest "$RAW_INVENTORY" \ + --output "$RESOLVED_MANIFEST" + RESOLVE_EXIT=$? + set -e + if [ "$RESOLVE_EXIT" -ne 0 ] && [ "$RESOLVE_EXIT" -ne 10 ]; then + exit "$RESOLVE_EXIT" + fi + + ELIGIBLE_COUNT=$(jq -r '[.candidates[] | select(.decision.eligible == true)] | length' "$RESOLVED_MANIFEST") + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$RESOLVED_MANIFEST") + test "$MANIFEST_DIGEST" != "null" + cp "$RESOLVED_MANIFEST" "$GITHUB_WORKSPACE/manifest.json" + { + echo "eligible-count=$ELIGIBLE_COUNT" + echo "source-commit=$EXPECTED_COMMIT" + echo "manifest-digest=$MANIFEST_DIGEST" + } >> "$GITHUB_OUTPUT" + + - name: Upload trusted candidate manifest + uses: actions/upload-artifact@v7 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: manifest.json + if-no-files-found: error + retention-days: 1 + +safe-outputs: + jobs: + apply-verified-unskips: + description: >- + Revalidate selected source sites and tracking items, apply only trusted + Ignore removals, require exact structured test execution evidence, and + open at most one draft pull request. + if: >- + needs.agent.result == 'success' && + needs.detection.result == 'success' && + needs.detection.outputs.detection_success == 'true' && + contains(needs.agent.outputs.output_types, 'apply_verified_unskips') + runs-on: ubuntu-latest + permissions: + contents: write + issues: read + pull-requests: write + inputs: + manifest_digest: + description: "Exact trusted manifest digest." + required: true + type: string + candidate_ids_json: + description: "JSON array of exact candidate IDs copied from the manifest." + required: true + type: string + steps: + - name: Checkout exact analyzed revision + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: true + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "8.0.x" + + - name: Restore trusted apply tool + working-directory: .github/workflows/unskip-closed-tests-tool + run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode + + - name: Download original trusted manifest + uses: actions/download-artifact@v8.0.1 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + + - name: Revalidate, edit, and verify selected candidates + id: apply + shell: bash + working-directory: .github/workflows/unskip-closed-tests-tool + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_COMMIT: ${{ github.sha }} + ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json + RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-result.json + RESULT_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-results + run: | + set -euo pipefail + set +e + dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- apply \ + --repo-root "$GITHUB_WORKSPACE" \ + --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ + --manifest "$ORIGINAL_MANIFEST" \ + --agent-output "$GH_AW_AGENT_OUTPUT" \ + --output "$RESULT_PATH" + APPLY_EXIT=$? + rm -rf bin obj + set -e + + if [ "$APPLY_EXIT" -eq 10 ]; then + git diff --quiet + echo "no-action=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ "$APPLY_EXIT" -ne 0 ]; then + exit "$APPLY_EXIT" + fi + + test -f "$RESULT_PATH" + jq -e \ + '.schema_version == "1" and .has_changes == true and (.changed_paths | length > 0)' \ + "$RESULT_PATH" >/dev/null + echo "no-action=false" >> "$GITHUB_OUTPUT" + echo "result-path=$RESULT_PATH" >> "$GITHUB_OUTPUT" + + - name: Publish one verified draft pull request + if: steps.apply.outputs.no-action == 'false' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_COMMIT: ${{ github.sha }} + RESULT_PATH: ${{ steps.apply.outputs.result-path }} + run: | + set -euo pipefail + + DEFAULT_BRANCH=$(gh api "repos/${EXPECTED_REPOSITORY}" --jq '.default_branch') + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + test "$CURRENT_HEAD" = "$EXPECTED_COMMIT" || + { echo "::notice::Default branch advanced; leaving verified changes unpublished."; exit 0; } + + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + test "$EXISTING" -eq 0 || + { echo "::notice::An unskip pull request is already open."; exit 0; } + + TITLE=$(jq -r '.pr_title' "$RESULT_PATH") + BODY_FILE="$RUNNER_TEMP/unskip-closed-tests-pr-body.md" + EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.txt" + ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.txt" + jq -r '.pr_body' "$RESULT_PATH" > "$BODY_FILE" + jq -r '.changed_paths[]' "$RESULT_PATH" | sort > "$EXPECTED_PATHS" + git diff --name-only --diff-filter=M | sort > "$ACTUAL_PATHS" + diff -u "$EXPECTED_PATHS" "$ACTUAL_PATHS" + while IFS= read -r path; do + test -n "$path" + test "${path#/}" = "$path" + case "/$path/" in + *"/../"*|*"/./"*) exit 20 ;; + esac + case "$path" in + *.cs) ;; + *) echo "::error::Unexpected changed path: $path"; exit 20 ;; + esac + git add -- "$path" + done < "$EXPECTED_PATHS" + + test -n "$(git diff --cached --name-only)" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Re-enable tests with resolved tracking items" + + BRANCH="automation/unskip-closed-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + git push origin "HEAD:refs/heads/$BRANCH" + + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + if [ "$CURRENT_HEAD" != "$EXPECTED_COMMIT" ]; then + git push origin --delete "$BRANCH" + echo "::notice::Default branch advanced before PR creation; removed the unpublished branch." + exit 0 + fi + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + if [ "$EXISTING" -ne 0 ]; then + git push origin --delete "$BRANCH" + echo "::notice::Another unskip pull request opened; removed the duplicate branch." + exit 0 + fi + + PR_URL=$(gh pr create \ + --repo "$EXPECTED_REPOSITORY" \ + --base "$DEFAULT_BRANCH" \ + --head "$BRANCH" \ + --draft \ + --title "$TITLE" \ + --body-file "$BODY_FILE") + + LIVE=$(gh pr view "$PR_URL" \ + --repo "$EXPECTED_REPOSITORY" \ + --json body,isDraft,headRefName,baseRefName,url) + test "$(printf '%s' "$LIVE" | jq -r '.isDraft')" = "true" + test "$(printf '%s' "$LIVE" | jq -r '.headRefName')" = "$BRANCH" + test "$(printf '%s' "$LIVE" | jq -r '.baseRefName')" = "$DEFAULT_BRANCH" + printf '%s' "$LIVE" | jq -r '.body' | grep -F '"); + body.AppendLine(); + body.AppendLine("## Verified unskips"); + body.AppendLine(); + foreach (Candidate candidate in retained) + { + body.Append("- `"); + body.Append(candidate.Path); + body.Append("` — "); + body.Append(string.Join(", ", candidate.Owner.TestFqns.Select(static fqn => $"`{fqn}`"))); + body.Append(" ("); + body.Append(string.Join(", ", candidate.CanonicalIssueReferences.Select(static reference => + $"[{reference.Canonical}]({reference.Url})"))); + body.AppendLine(")"); + } + + body.AppendLine(); + body.AppendLine("Each retained edit was verified independently by the configured trusted command and exact TRX FQN mapping."); + if (reverted.Count > 0) + { + body.AppendLine(); + body.AppendLine($"The helper reverted {reverted.Count} candidate(s) that did not satisfy verification."); + } + + return body.ToString().TrimEnd(); + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/ConfigLoader.cs b/.github/workflows/unskip-closed-tests-tool/ConfigLoader.cs new file mode 100644 index 0000000000..da4c971b97 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/ConfigLoader.cs @@ -0,0 +1,206 @@ +using System.Text.Json; + +namespace UnskipClosedTests.Tool; + +internal static class ConfigLoader +{ + private static readonly HashSet AllowedProperties = + [ + "schema_version", + "source_roots", + "excluded_globs", + "generated_globs", + "ignore_attribute_names", + "test_attribute_names", + "verification", + ]; + + public static ToolConfig Load(string path) + { + using JsonDocument document = JsonSupport.ReadDocument(path); + JsonElement root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + throw new ContractException("Config root must be an object."); + } + + RejectUnknownProperties(root, AllowedProperties, "config"); + ToolConfig config = new() + { + SchemaVersion = RequiredString(root, "schema_version"), + SourceRoots = RequiredStringArray(root, "source_roots"), + ExcludedGlobs = OptionalStringArray(root, "excluded_globs"), + GeneratedGlobs = OptionalStringArray(root, "generated_globs"), + IgnoreAttributeNames = RequiredStringArray(root, "ignore_attribute_names"), + TestAttributeNames = RequiredStringArray(root, "test_attribute_names"), + }; + + if (!root.TryGetProperty("verification", out JsonElement verification) || + verification.ValueKind != JsonValueKind.Object) + { + throw new ContractException("verification must be an object."); + } + + RejectUnknownProperties(verification, ["command", "timeout_seconds"], "verification"); + config.VerificationCommand = RequiredStringArray(verification, "command"); + config.VerificationTimeoutSeconds = RequiredPositiveInt(verification, "timeout_seconds"); + + Validate(config); + return config; + } + + public static string Digest(ToolConfig config) => JsonSupport.CanonicalDigest(config); + + private static void Validate(ToolConfig config) + { + if (config.SchemaVersion != "1") + { + throw new ContractException($"Unsupported config schema_version '{config.SchemaVersion}'."); + } + + if (config.SourceRoots.Count == 0) + { + throw new ContractException("source_roots must contain at least one path."); + } + + if (config.IgnoreAttributeNames.Count == 0 || config.TestAttributeNames.Count == 0) + { + throw new ContractException("ignore_attribute_names and test_attribute_names must not be empty."); + } + + if (config.VerificationCommand.Count == 0) + { + throw new ContractException("verification.command must not be empty."); + } + + ValidateUniqueNonEmpty(config.SourceRoots, "source_roots"); + ValidateUniqueNonEmpty(config.ExcludedGlobs, "excluded_globs"); + ValidateUniqueNonEmpty(config.GeneratedGlobs, "generated_globs"); + ValidateUniqueNonEmpty(config.IgnoreAttributeNames, "ignore_attribute_names"); + ValidateUniqueNonEmpty(config.TestAttributeNames, "test_attribute_names"); + ValidateUniqueNonEmpty(config.VerificationCommand, "verification.command", requireUnique: false); + + foreach (string root in config.SourceRoots) + { + PathRules.ValidateRelativePath(root, "source root"); + } + + foreach (string glob in config.ExcludedGlobs.Concat(config.GeneratedGlobs)) + { + if (Path.IsPathRooted(glob) || glob.Contains('\\')) + { + throw new ContractException($"Glob '{glob}' must be repository-relative and use '/' separators."); + } + + if (glob.Split('/').Any(static segment => segment == "..")) + { + throw new ContractException($"Glob '{glob}' contains traversal."); + } + } + + foreach (string name in config.IgnoreAttributeNames.Concat(config.TestAttributeNames)) + { + if (!IsAttributeName(name)) + { + throw new ContractException($"Attribute name '{name}' is not a simple or qualified C# identifier."); + } + } + } + + private static bool IsAttributeName(string value) + { + string[] pieces = value.Split('.'); + return pieces.Length > 0 && pieces.All(static piece => + piece.Length > 0 && + (char.IsLetter(piece[0]) || piece[0] == '_') && + piece.Skip(1).All(static character => char.IsLetterOrDigit(character) || character == '_')); + } + + private static void ValidateUniqueNonEmpty(List values, string name, bool requireUnique = true) + { + if (values.Any(static value => string.IsNullOrWhiteSpace(value))) + { + throw new ContractException($"{name} contains an empty value."); + } + + if (requireUnique && values.Distinct(StringComparer.Ordinal).Count() != values.Count) + { + throw new ContractException($"{name} contains duplicate values."); + } + } + + private static void RejectUnknownProperties(JsonElement element, HashSet allowed, string context) + { + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + throw new ContractException($"Unknown {context} property '{property.Name}'."); + } + } + } + + private static string RequiredString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind != JsonValueKind.String) + { + throw new ContractException($"{name} must be a string."); + } + + return value.GetString()!; + } + + private static List RequiredStringArray(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value)) + { + throw new ContractException($"{name} is required."); + } + + return ReadStringArray(value, name); + } + + private static List OptionalStringArray(JsonElement element, string name) => + element.TryGetProperty(name, out JsonElement value) ? ReadStringArray(value, name) : []; + + private static List ReadStringArray(JsonElement value, string name) + { + if (value.ValueKind != JsonValueKind.Array) + { + throw new ContractException($"{name} must be an array."); + } + + List result = []; + foreach (JsonElement item in value.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + throw new ContractException($"{name} must contain only strings."); + } + + result.Add(item.GetString()!); + } + + return result; + } + + private static int RequiredPositiveInt(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value)) + { + throw new ContractException($"{name} is required."); + } + + return ReadPositiveInt(value, name); + } + + private static int ReadPositiveInt(JsonElement value, string name) + { + if (value.ValueKind != JsonValueKind.Number || !value.TryGetInt32(out int result) || result <= 0) + { + throw new ContractException($"{name} must be a positive 32-bit integer."); + } + + return result; + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Directory.Build.props b/.github/workflows/unskip-closed-tests-tool/Directory.Build.props new file mode 100644 index 0000000000..058246e408 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Directory.Build.props @@ -0,0 +1 @@ + diff --git a/.github/workflows/unskip-closed-tests-tool/Directory.Build.targets b/.github/workflows/unskip-closed-tests-tool/Directory.Build.targets new file mode 100644 index 0000000000..058246e408 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Directory.Build.targets @@ -0,0 +1 @@ + diff --git a/.github/workflows/unskip-closed-tests-tool/Directory.Packages.props b/.github/workflows/unskip-closed-tests-tool/Directory.Packages.props new file mode 100644 index 0000000000..5f9708a97f --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Directory.Packages.props @@ -0,0 +1,5 @@ + + + false + + diff --git a/.github/workflows/unskip-closed-tests-tool/GitRepository.cs b/.github/workflows/unskip-closed-tests-tool/GitRepository.cs new file mode 100644 index 0000000000..8922c5a0a4 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/GitRepository.cs @@ -0,0 +1,168 @@ +using System.Diagnostics; +using System.Text; +using System.Text.RegularExpressions; + +namespace UnskipClosedTests.Tool; + +internal sealed class GitRepository +{ + private static readonly Regex GitHubRemotePattern = new( + @"(?:github\.com[:/])(?[^/:\s]+)/(?[^/\s]+?)(?:\.git)?$", + RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking); + + private GitRepository(string root, string commit, string objectFormat, string repository) + { + Root = root; + Commit = commit; + ObjectFormat = objectFormat; + Repository = repository; + } + + public string Root { get; } + public string Commit { get; } + public string ObjectFormat { get; } + public string Repository { get; } + + public static GitRepository Open(string requestedRoot, string? repositoryOverride) + { + string root = RunGit(requestedRoot, ["rev-parse", "--show-toplevel"]).Trim(); + if (root.Length == 0) + { + throw new ContractException("Could not determine the repository root."); + } + + root = Path.GetFullPath(root); + string requested = Path.GetFullPath(requestedRoot); + if (!string.Equals(root, requested, StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException($"--repo-root must be the exact git repository root '{root}'."); + } + + string commit = RunGit(root, ["rev-parse", "HEAD"]).Trim(); + string objectFormat = RunGit(root, ["rev-parse", "--show-object-format"]).Trim(); + if (objectFormat is not ("sha1" or "sha256")) + { + throw new ContractException($"Unsupported git object format '{objectFormat}'."); + } + + string repository = repositoryOverride is null + ? InferRepository(root) + : ValidateRepository(repositoryOverride); + return new GitRepository(root, commit, objectFormat, repository); + } + + public string HeadBlobOid(string path) + { + string normalized = PathRules.ValidateRelativePath(path, "source path"); + string output = RunGit(Root, ["ls-tree", Commit, "--", normalized]).Trim(); + if (output.Length == 0) + { + throw new ContractException($"Source path '{normalized}' is not tracked at commit {Commit}."); + } + + string[] tabParts = output.Split('\t'); + string[] metadata = tabParts[0].Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (metadata.Length != 3 || metadata[1] != "blob") + { + throw new ContractException($"Source path '{normalized}' is not a regular tracked blob."); + } + + if (metadata[0] == "120000") + { + throw new ContractException($"Source path '{normalized}' is a git symlink."); + } + + return metadata[2]; + } + + public byte[] HeadBytes(string path) + { + string normalized = PathRules.ValidateRelativePath(path, "source path"); + return RunGitBytes(Root, ["show", $"{Commit}:{normalized}"]); + } + + public void RequireWorktreeMatchesHead(string path, byte[] bytes) + { + _ = bytes; + string normalized = PathRules.ValidateRelativePath(path, "source path"); + string status = RunGit( + Root, + ["status", "--porcelain=v1", "--untracked-files=no", "--", normalized]).Trim(); + if (status.Length != 0) + { + throw new ContractException($"Source path '{path}' does not match checked-out commit {Commit}."); + } + } + + public string MetadataDirectory() + { + string value = RunGit(Root, ["rev-parse", "--git-dir"]).Trim(); + return Path.GetFullPath(Path.IsPathRooted(value) ? value : Path.Combine(Root, value)); + } + + private static string InferRepository(string root) + { + string remote = RunGit(root, ["config", "--get", "remote.origin.url"], allowFailure: true).Trim(); + Match match = GitHubRemotePattern.Match(remote); + if (!match.Success) + { + throw new ContractException("Could not infer owner/repo from remote.origin.url; pass --repository."); + } + + return ValidateRepository($"{match.Groups["owner"].Value}/{match.Groups["repo"].Value}"); + } + + private static string ValidateRepository(string repository) + { + string[] parts = repository.Split('/'); + if (parts.Length != 2 || parts.Any(static part => part.Length == 0 || part is "." or "..")) + { + throw new ContractException($"Repository '{repository}' must be owner/repo."); + } + + return $"{parts[0].ToLowerInvariant()}/{parts[1].ToLowerInvariant()}"; + } + + private static string RunGit(string workingDirectory, IReadOnlyList arguments, bool allowFailure = false) => + Encoding.UTF8.GetString(RunGitBytes(workingDirectory, arguments, allowFailure)); + + private static byte[] RunGitBytes(string workingDirectory, IReadOnlyList arguments, bool allowFailure = false) + { + ProcessStartInfo startInfo = new("git") + { + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (string argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + try + { + using Process process = Process.Start(startInfo) + ?? throw new InfrastructureException("Could not start git."); + using MemoryStream output = new(); + process.StandardOutput.BaseStream.CopyTo(output); + string error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (process.ExitCode != 0 && !allowFailure) + { + throw new ContractException($"git {string.Join(' ', arguments)} failed: {error.Trim()}"); + } + + return output.ToArray(); + } + catch (ContractException) + { + throw; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidOperationException) + { + throw new InfrastructureException("Could not invoke git.", ex); + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/InventoryEngine.cs b/.github/workflows/unskip-closed-tests-tool/InventoryEngine.cs new file mode 100644 index 0000000000..992806ef69 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/InventoryEngine.cs @@ -0,0 +1,587 @@ +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; + +namespace UnskipClosedTests.Tool; + +internal static partial class InventoryEngine +{ + private sealed record ParsedFile( + string Path, + string FullPath, + byte[] Bytes, + string BlobOid, + SyntaxTree Tree, + CompilationUnitSyntax Root); + + private sealed record PendingCandidate( + ParsedFile File, + AttributeSyntax Attribute, + OwnerIdentity Owner, + string StableOwnerId, + List References, + List StructuralDeferrals); + + public static Manifest Create(string requestedRoot, string? repositoryOverride, ToolConfig config) + { + GitRepository repository = GitRepository.Open(requestedRoot, repositoryOverride); + List files = LoadFiles(repository, config); + Dictionary typeDeclarationCounts = CountTypeDeclarations(files); + List pending = []; + + foreach (ParsedFile file in files) + { + foreach (AttributeSyntax attribute in file.Root.DescendantNodes().OfType()) + { + if (!AttributeMatches(attribute, config.IgnoreAttributeNames)) + { + continue; + } + + List references = ExtractReferences(attribute, repository.Repository); + if (references.Count == 0) + { + continue; + } + + if (attribute.FirstAncestorOrSelf() is MethodDeclarationSyntax method && + method.AttributeLists.Any(list => list.Span.Contains(attribute.Span))) + { + OwnerIdentity owner = CreateMethodOwner(method, config); + List deferrals = []; + if (owner.TestFqns.Count == 0) + { + deferrals.Add("no_enumerated_tests"); + } + if (HasGeneratedMarker(method)) + { + deferrals.Add("generated_declaration"); + } + + string stableOwnerId = StableOwnerId(repository.Repository, file.Path, owner, method); + pending.Add(new PendingCandidate(file, attribute, owner, stableOwnerId, references, deferrals)); + continue; + } + + if (attribute.FirstAncestorOrSelf() is ClassDeclarationSyntax type && + type.AttributeLists.Any(list => list.Span.Contains(attribute.Span))) + { + (OwnerIdentity owner, List deferrals) = + CreateClassOwner(type, config, typeDeclarationCounts); + if (HasGeneratedMarker(type)) + { + deferrals.Add("generated_declaration"); + } + + string stableOwnerId = StableOwnerId(repository.Repository, file.Path, owner, type); + pending.Add(new PendingCandidate(file, attribute, owner, stableOwnerId, references, deferrals)); + } + } + } + + List candidates = []; + foreach (IGrouping ownerGroup in pending + .OrderBy(static item => item.File.Path, StringComparer.Ordinal) + .ThenBy(static item => item.Attribute.SpanStart) + .GroupBy(static item => item.StableOwnerId, StringComparer.Ordinal)) + { + int ordinal = 0; + foreach (PendingCandidate item in ownerGroup) + { + ordinal++; + FileLinePositionSpan lineSpan = item.Attribute.GetLocation().GetLineSpan(); + SourceSpan sourceSpan = new() + { + Start = item.Attribute.SpanStart, + Length = item.Attribute.Span.Length, + StartLine = lineSpan.StartLinePosition.Line + 1, + StartColumn = lineSpan.StartLinePosition.Character + 1, + EndLine = lineSpan.EndLinePosition.Line + 1, + EndColumn = lineSpan.EndLinePosition.Character + 1, + }; + string attributeText = item.File.Root.SyntaxTree.GetText().ToString(item.Attribute.Span); + string candidateId = JsonSupport.Sha256( + $"candidate-v1\0{repository.Repository}\0{item.File.Path}\0{item.StableOwnerId}\0" + + $"{item.File.BlobOid}\0{sourceSpan.Start}:{sourceSpan.Length}\0{ordinal}"); + + candidates.Add(new Candidate + { + CandidateId = candidateId, + StableOwnerId = item.StableOwnerId, + Path = item.File.Path, + BlobOid = item.File.BlobOid, + SourceSha256 = JsonSupport.Sha256(item.File.Bytes), + AttributeSpan = sourceSpan, + AttributeTextSha256 = JsonSupport.Sha256(attributeText), + Owner = item.Owner, + CanonicalIssueReferences = item.References, + Decision = new CandidateDecision + { + Eligible = false, + Deferrals = item.StructuralDeferrals.Order(StringComparer.Ordinal).ToList(), + }, + }); + } + } + + candidates = candidates + .OrderBy(static candidate => candidate.Path, StringComparer.Ordinal) + .ThenBy(static candidate => candidate.AttributeSpan.Start) + .ToList(); + Manifest manifest = new() + { + Repository = repository.Repository, + SourceCommit = repository.Commit, + GitObjectFormat = repository.ObjectFormat, + ConfigDigest = ConfigLoader.Digest(config), + CandidateCount = candidates.Count, + Candidates = candidates, + }; + manifest.ManifestDigest = JsonSupport.ManifestDigest(manifest); + return manifest; + } + + private static List LoadFiles(GitRepository repository, ToolConfig config) + { + Dictionary paths = new(StringComparer.Ordinal); + foreach (string configuredRoot in config.SourceRoots) + { + string normalizedRoot = PathRules.ValidateRelativePath(configuredRoot, "source root"); + string fullRoot = PathRules.ResolveInsideRoot(repository.Root, normalizedRoot, "source root"); + if (File.Exists(fullRoot)) + { + if (!normalizedRoot.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException($"Source root '{normalizedRoot}' is not a C# file."); + } + + if (PathRules.MatchesAnyGlob(normalizedRoot, config.ExcludedGlobs.Concat(config.GeneratedGlobs))) + { + throw new ContractException($"Explicit source root '{normalizedRoot}' is excluded or generated."); + } + + paths[normalizedRoot] = fullRoot; + continue; + } + + if (!Directory.Exists(fullRoot)) + { + continue; + } + + PathRules.RejectReparsePoints(repository.Root, fullRoot); + foreach (string file in Directory.EnumerateFiles(fullRoot, "*", SearchOption.AllDirectories)) + { + string extension = Path.GetExtension(file); + if (!string.Equals(extension, ".cs", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + string relative = Path.GetRelativePath(repository.Root, file).Replace('\\', '/'); + relative = PathRules.ValidateRelativePath(relative, "source path"); + if (PathRules.MatchesAnyGlob(relative, config.ExcludedGlobs.Concat(config.GeneratedGlobs))) + { + continue; + } + + paths[relative] = file; + } + } + + List result = []; + foreach ((string relative, string fullPath) in paths.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + PathRules.RejectReparsePoints(repository.Root, fullPath); + byte[] bytes; + try + { + bytes = File.ReadAllBytes(fullPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + throw new InfrastructureException($"Could not read source path '{relative}'.", ex); + } + + string blobOid = repository.HeadBlobOid(relative); + repository.RequireWorktreeMatchesHead(relative, bytes); + string source; + try + { + int offset = bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF ? 3 : 0; + source = new UTF8Encoding(false, true).GetString(bytes, offset, bytes.Length - offset); + } + catch (DecoderFallbackException ex) + { + throw new ContractException($"Source path '{relative}' is not valid UTF-8: {ex.Message}"); + } + + string prefix = source[..Math.Min(source.Length, 2048)]; + if (prefix.Contains(" errors = tree.GetDiagnostics() + .Where(static diagnostic => diagnostic.Severity == DiagnosticSeverity.Error) + .ToList(); + if (errors.Count > 0) + { + throw new ContractException( + $"Source path '{relative}' has C# parse errors: {string.Join("; ", errors.Take(3))}"); + } + + result.Add(new ParsedFile(relative, fullPath, bytes, blobOid, tree, tree.GetCompilationUnitRoot())); + } + + return result; + } + + private static Dictionary CountTypeDeclarations(IEnumerable files) + { + Dictionary counts = new(StringComparer.Ordinal); + foreach (TypeDeclarationSyntax declaration in files.SelectMany(static file => + file.Root.DescendantNodes().OfType())) + { + string fqn = TypeFqn(declaration); + counts[fqn] = counts.GetValueOrDefault(fqn) + 1; + } + + return counts; + } + + private static OwnerIdentity CreateMethodOwner(MethodDeclarationSyntax method, ToolConfig config) + { + TypeDeclarationSyntax? type = method.Ancestors().OfType().FirstOrDefault(); + if (type is null) + { + throw new ContractException("An Ignore attribute on a method has no actual containing type."); + } + + string typeFqn = TypeFqn(type); + string signature = MethodSignature(method); + bool isTest = method.AttributeLists.SelectMany(static list => list.Attributes) + .Any(attribute => AttributeMatches(attribute, config.TestAttributeNames)); + return new OwnerIdentity + { + Kind = "method", + Namespace = NamespaceName(type), + ContainingTypes = ContainingTypeNames(type), + TypeFqn = typeFqn, + DeclarationId = MethodDeclarationId(method), + MethodName = method.Identifier.ValueText, + MethodSignature = signature, + TestFqns = isTest ? [$"{typeFqn}.{method.Identifier.ValueText}"] : [], + }; + } + + private static (OwnerIdentity Owner, List Deferrals) CreateClassOwner( + ClassDeclarationSyntax type, + ToolConfig config, + IReadOnlyDictionary declarationCounts) + { + string typeFqn = TypeFqn(type); + List deferrals = []; + List containingTypes = ContainingTypeNames(type); + if (containingTypes.Count > 1) + { + deferrals.Add("class_is_nested"); + } + + if (type.Modifiers.Any(SyntaxKind.PartialKeyword)) + { + deferrals.Add("class_is_partial"); + } + + if (type.BaseList is not null && type.BaseList.Types.Count > 0) + { + deferrals.Add("class_has_base_types"); + } + + if (declarationCounts.GetValueOrDefault(typeFqn) != 1) + { + deferrals.Add("duplicate_type_declarations"); + } + + List tests = type.Members + .OfType() + .Where(method => method.AttributeLists.SelectMany(static list => list.Attributes) + .Any(attribute => AttributeMatches(attribute, config.TestAttributeNames))) + .Select(method => $"{typeFqn}.{method.Identifier.ValueText}") + .Order(StringComparer.Ordinal) + .ToList(); + if (tests.Count == 0) + { + deferrals.Add("no_enumerated_tests"); + } + + if (tests.Distinct(StringComparer.Ordinal).Count() != tests.Count) + { + deferrals.Add("ambiguous_test_fqns"); + } + + OwnerIdentity owner = new() + { + Kind = "class", + Namespace = NamespaceName(type), + ContainingTypes = containingTypes, + TypeFqn = typeFqn, + DeclarationId = $"T:{typeFqn}", + TestFqns = tests.Distinct(StringComparer.Ordinal).ToList(), + }; + return (owner, deferrals.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList()); + } + + private static string StableOwnerId( + string repository, + string path, + OwnerIdentity owner, + MemberDeclarationSyntax declaration) + { + int declarationOrdinal = declaration switch + { + MethodDeclarationSyntax method => method.SyntaxTree.GetRoot() + .DescendantNodes() + .OfType() + .Where(candidate => string.Equals( + MethodDeclarationId(candidate), + owner.DeclarationId, + StringComparison.Ordinal)) + .Count(candidate => candidate.SpanStart < method.SpanStart) + 1, + TypeDeclarationSyntax type => type.SyntaxTree.GetRoot() + .DescendantNodes() + .OfType() + .Where(candidate => string.Equals( + $"T:{TypeFqn(candidate)}", + owner.DeclarationId, + StringComparison.Ordinal)) + .Count(candidate => candidate.SpanStart < type.SpanStart) + 1, + _ => throw new ContractException("Unsupported owner declaration kind."), + }; + return JsonSupport.Sha256( + $"owner-v1\0{repository}\0{path}\0{owner.DeclarationId}\0{declarationOrdinal}"); + } + + private static bool HasGeneratedMarker(MemberDeclarationSyntax declaration) => + HasDirectGeneratedMarker(declaration) || + declaration.Ancestors().OfType().Any(HasDirectGeneratedMarker); + + private static bool HasDirectGeneratedMarker(MemberDeclarationSyntax declaration) => + declaration.AttributeLists + .SelectMany(static list => list.Attributes) + .Any(static attribute => + { + string name = attribute.Name.WithoutTrivia().ToFullString() + .Replace("global::", "", StringComparison.Ordinal) + .Split('.') + .Last(); + if (name.EndsWith("Attribute", StringComparison.Ordinal)) + { + name = name[..^"Attribute".Length]; + } + + return name is "GeneratedCode" or "CompilerGenerated"; + }); + + private static string MethodSignature(MethodDeclarationSyntax method) + { + string explicitInterface = method.ExplicitInterfaceSpecifier is null + ? "" + : $"{method.ExplicitInterfaceSpecifier.Name.WithoutTrivia().ToFullString()}."; + string arity = method.TypeParameterList is null ? "" : $"`{method.TypeParameterList.Parameters.Count}"; + string parameters = string.Join(",", + method.ParameterList.Parameters.Select(static parameter => + $"{parameter.Modifiers.ToFullString().Trim()}:{parameter.Type?.WithoutTrivia().ToFullString() ?? "?"}")); + return $"{explicitInterface}{method.Identifier.ValueText}{arity}({parameters})"; + } + + private static string MethodDeclarationId(MethodDeclarationSyntax method) + { + TypeDeclarationSyntax? type = method.Ancestors().OfType().FirstOrDefault(); + if (type is null) + { + throw new ContractException("A method owner has no actual containing type."); + } + + return $"M:{TypeFqn(type)}.{MethodSignature(method)}"; + } + + private static string TypeFqn(TypeDeclarationSyntax type) + { + List parts = []; + string namespaceName = NamespaceName(type); + if (namespaceName.Length > 0) + { + parts.Add(namespaceName); + } + + parts.AddRange(ContainingTypeNames(type)); + return string.Join('.', parts); + } + + private static string NamespaceName(SyntaxNode node) => + string.Join('.', + node.Ancestors() + .OfType() + .Reverse() + .Select(static declaration => declaration.Name.WithoutTrivia().ToFullString())); + + private static List ContainingTypeNames(TypeDeclarationSyntax type) => + type.AncestorsAndSelf() + .OfType() + .Reverse() + .Select(static declaration => + declaration.TypeParameterList is null + ? declaration.Identifier.ValueText + : $"{declaration.Identifier.ValueText}`{declaration.TypeParameterList.Parameters.Count}") + .ToList(); + + internal static bool AttributeMatches(AttributeSyntax attribute, IEnumerable configuredNames) + { + string actual = attribute.Name.WithoutTrivia().ToFullString().Replace("global::", "", StringComparison.Ordinal); + string actualShort = actual.Split('.').Last(); + return configuredNames.Any(configured => + { + string normalized = configured.EndsWith("Attribute", StringComparison.Ordinal) + ? configured[..^"Attribute".Length] + : configured; + string actualNormalized = actual.EndsWith("Attribute", StringComparison.Ordinal) + ? actual[..^"Attribute".Length] + : actual; + string shortNormalized = actualShort.EndsWith("Attribute", StringComparison.Ordinal) + ? actualShort[..^"Attribute".Length] + : actualShort; + return normalized.Contains('.', StringComparison.Ordinal) + ? string.Equals(normalized, actualNormalized, StringComparison.Ordinal) + : string.Equals(normalized, shortNormalized, StringComparison.Ordinal); + }); + } + + private static List ExtractReferences(AttributeSyntax attribute, string currentRepository) + { + List references = []; + if (attribute.ArgumentList is null) + { + return references; + } + + foreach (AttributeArgumentSyntax argument in attribute.ArgumentList.Arguments) + { + bool supportedName = argument.NameEquals is null || + string.Equals(argument.NameEquals.Name.Identifier.ValueText, "IgnoreMessage", StringComparison.Ordinal); + if (!supportedName || argument.NameColon is not null || + argument.Expression is not LiteralExpressionSyntax literal || + !literal.IsKind(SyntaxKind.StringLiteralExpression)) + { + continue; + } + + string value = literal.Token.ValueText; + references.AddRange(ParseReferences(value, currentRepository)); + } + + return references + .GroupBy(static reference => reference.Canonical, StringComparer.Ordinal) + .Select(static group => group.First()) + .OrderBy(static reference => reference.Canonical, StringComparer.Ordinal) + .ToList(); + } + + private static IEnumerable ParseReferences(string value, string currentRepository) + { + List<(int Start, int Length, string Owner, string Repo, int Number, string Kind)> matches = []; + foreach (Match match in FullReferenceRegex().Matches(value)) + { + matches.Add(( + match.Index, + match.Length, + match.Groups["owner"].Value, + match.Groups["repo"].Value, + int.Parse(match.Groups["number"].Value, System.Globalization.CultureInfo.InvariantCulture), + match.Groups["kind"].Value.Equals("pull", StringComparison.OrdinalIgnoreCase) ? "pull_request" : "issue")); + } + + foreach (Match match in QualifiedReferenceRegex().Matches(value)) + { + if (matches.Any(existing => RangesOverlap(existing.Start, existing.Length, match.Index, match.Length))) + { + continue; + } + + matches.Add(( + match.Index, + match.Length, + match.Groups["owner"].Value, + match.Groups["repo"].Value, + int.Parse(match.Groups["number"].Value, System.Globalization.CultureInfo.InvariantCulture), + "unknown")); + } + + string[] current = currentRepository.Split('/'); + foreach (Match match in BareReferenceRegex().Matches(value)) + { + if (matches.Any(existing => RangesOverlap(existing.Start, existing.Length, match.Index, match.Length))) + { + continue; + } + + matches.Add(( + match.Index, + match.Length, + current[0], + current[1], + int.Parse(match.Groups["number"].Value, System.Globalization.CultureInfo.InvariantCulture), + "unknown")); + } + + foreach ((_, _, string ownerValue, string repoValue, int number, string kind) in matches.OrderBy(static item => item.Start)) + { + if (number <= 0) + { + continue; + } + + string owner = ownerValue.ToLowerInvariant(); + string repo = repoValue.ToLowerInvariant(); + string pathKind = kind == "pull_request" ? "pull" : "issues"; + yield return new IssueReference + { + Kind = kind, + Owner = owner, + Repo = repo, + Number = number, + Canonical = $"{owner}/{repo}#{number}", + Url = $"https://github.com/{owner}/{repo}/{pathKind}/{number}", + Eligibility = false, + State = "unknown", + StateReason = "unresolved", + }; + } + } + + private static bool RangesOverlap(int firstStart, int firstLength, int secondStart, int secondLength) => + firstStart < secondStart + secondLength && secondStart < firstStart + firstLength; + + [GeneratedRegex( + @"https://github\.com/(?[A-Za-z0-9_.-]+)/(?[A-Za-z0-9_.-]+)/(?issues|pull)/(?[1-9][0-9]*)", + RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking)] + private static partial Regex FullReferenceRegex(); + + [GeneratedRegex( + @"(?[A-Za-z0-9_.-]+)/(?[A-Za-z0-9_.-]+)#(?[1-9][0-9]*)(?![0-9])", + RegexOptions.CultureInvariant)] + private static partial Regex QualifiedReferenceRegex(); + + [GeneratedRegex( + @"(?[1-9][0-9]*)(?![0-9])", + RegexOptions.CultureInvariant)] + private static partial Regex BareReferenceRegex(); +} diff --git a/.github/workflows/unskip-closed-tests-tool/IssueResolver.cs b/.github/workflows/unskip-closed-tests-tool/IssueResolver.cs new file mode 100644 index 0000000000..5281b1e624 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/IssueResolver.cs @@ -0,0 +1,483 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; + +namespace UnskipClosedTests.Tool; + +internal static class IssueResolver +{ + private static readonly HashSet StructuralDeferrals = + [ + "no_enumerated_tests", + "class_is_nested", + "class_is_partial", + "class_has_base_types", + "duplicate_type_declarations", + "ambiguous_test_fqns", + "generated_declaration", + ]; + + public static async Task ResolveAsync(Manifest input, string? evidencePath) + { + ManifestValidator.Validate(input); + Manifest manifest = Clone(input); + IReferenceEvidenceProvider provider = evidencePath is null + ? new GitHubReferenceEvidenceProvider() + : FixtureReferenceEvidenceProvider.Load(evidencePath); + + Dictionary cache = new(StringComparer.Ordinal); + foreach (Candidate candidate in manifest.Candidates) + { + List deferrals = candidate.Decision.Deferrals + .Where(StructuralDeferrals.Contains) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToList(); + List resolvedReferences = []; + foreach (IssueReference reference in candidate.CanonicalIssueReferences) + { + if (!cache.TryGetValue(reference.Canonical, out EvidenceReference? evidence)) + { + evidence = await provider.GetAsync(reference); + cache.Add(reference.Canonical, evidence); + } + + IssueReference resolved = ResolveReference(reference, evidence); + resolvedReferences.Add(resolved); + if (!resolved.Eligibility) + { + deferrals.Add($"reference_not_eligible:{resolved.Canonical}:{resolved.StateReason}"); + } + } + + candidate.CanonicalIssueReferences = resolvedReferences + .OrderBy(static reference => reference.Canonical, StringComparer.Ordinal) + .ToList(); + if (candidate.Owner.TestFqns.Count == 0 && !deferrals.Contains("no_enumerated_tests", StringComparer.Ordinal)) + { + deferrals.Add("no_enumerated_tests"); + } + + candidate.Decision = new CandidateDecision + { + Eligible = deferrals.Count == 0 && + candidate.CanonicalIssueReferences.Count > 0 && + candidate.CanonicalIssueReferences.All(static reference => reference.Eligibility), + Deferrals = deferrals.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList(), + }; + } + + manifest.ManifestDigest = ""; + manifest.ManifestDigest = JsonSupport.ManifestDigest(manifest); + return manifest; + } + + private static IssueReference ResolveReference(IssueReference original, EvidenceReference evidence) + { + if (!string.Equals(evidence.Canonical, original.Canonical, StringComparison.Ordinal)) + { + throw new ContractException( + $"Evidence canonical '{evidence.Canonical}' does not match requested reference '{original.Canonical}'."); + } + + string kind = NormalizeKind(evidence.Kind.Length == 0 ? original.Kind : evidence.Kind); + if (!evidence.Accessible) + { + return Copy(original, kind, false, "inaccessible", "inaccessible", null); + } + + if (kind == "unknown") + { + return Copy(original, kind, false, "unknown", "unknown_reference_kind", null); + } + + string state = evidence.State.ToLowerInvariant(); + string stateReason = evidence.StateReason.ToLowerInvariant(); + if (kind == "issue") + { + bool eligible = state == "closed" && stateReason == "completed"; + string reason = eligible + ? "completed" + : state == "open" + ? "open" + : stateReason == "not_planned" + ? "not_planned" + : "not_completed"; + return Copy(original, kind, eligible, state, reason, null); + } + + bool merged = !string.IsNullOrWhiteSpace(evidence.MergedAt); + if (merged && !DateTimeOffset.TryParse( + evidence.MergedAt, + System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.RoundtripKind, + out _)) + { + throw new ContractException($"Pull request evidence '{evidence.Canonical}' has malformed merged_at."); + } + + return Copy( + original, + kind, + merged, + state.Length == 0 ? (merged ? "closed" : "unknown") : state, + merged ? "merged" : "not_merged", + evidence.MergedAt); + } + + private static IssueReference Copy( + IssueReference original, + string kind, + bool eligible, + string state, + string stateReason, + string? mergedAt) + { + string pathKind = kind == "pull_request" ? "pull" : "issues"; + return new IssueReference + { + Kind = kind, + Owner = original.Owner, + Repo = original.Repo, + Number = original.Number, + Canonical = original.Canonical, + Url = $"https://github.com/{original.Owner}/{original.Repo}/{pathKind}/{original.Number}", + Eligibility = eligible, + State = state, + StateReason = stateReason, + MergedAt = mergedAt, + }; + } + + private static string NormalizeKind(string kind) => kind.ToLowerInvariant() switch + { + "issue" => "issue", + "pull" or "pr" or "pull_request" => "pull_request", + "unknown" or "" => "unknown", + _ => throw new ContractException($"Unsupported evidence kind '{kind}'."), + }; + + private static Manifest Clone(Manifest input) + { + string json = JsonSerializer.Serialize(input, JsonSupport.Options); + return JsonSerializer.Deserialize(json, JsonSupport.Options) + ?? throw new InfrastructureException("Could not clone manifest."); + } + + private interface IReferenceEvidenceProvider + { + Task GetAsync(IssueReference reference); + } + + private sealed class FixtureReferenceEvidenceProvider( + IReadOnlyDictionary references) : IReferenceEvidenceProvider + { + public static FixtureReferenceEvidenceProvider Load(string path) + { + using JsonDocument document = JsonSupport.ReadDocument(path); + JsonElement root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + throw new ContractException("GitHub evidence root must be an object."); + } + + foreach (JsonProperty property in root.EnumerateObject()) + { + if (property.Name is not ("schema_version" or "references")) + { + throw new ContractException($"Unknown GitHub evidence property '{property.Name}'."); + } + } + + if (!root.TryGetProperty("schema_version", out JsonElement schema) || + schema.ValueKind != JsonValueKind.String || + schema.GetString() != "1") + { + throw new ContractException("GitHub evidence schema_version must be '1'."); + } + + if (!root.TryGetProperty("references", out JsonElement referencesElement)) + { + throw new ContractException("GitHub evidence references is required."); + } + + Dictionary references = new(StringComparer.Ordinal); + if (referencesElement.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement item in referencesElement.EnumerateArray()) + { + EvidenceReference evidence = ParseEvidence(item, null); + if (!references.TryAdd(evidence.Canonical, evidence)) + { + throw new ContractException($"Duplicate GitHub evidence '{evidence.Canonical}'."); + } + } + } + else if (referencesElement.ValueKind == JsonValueKind.Object) + { + foreach (JsonProperty property in referencesElement.EnumerateObject()) + { + EvidenceReference evidence = ParseEvidence(property.Value, property.Name); + if (!references.TryAdd(evidence.Canonical, evidence)) + { + throw new ContractException($"Duplicate GitHub evidence '{evidence.Canonical}'."); + } + } + } + else + { + throw new ContractException("GitHub evidence references must be an array or object."); + } + + return new FixtureReferenceEvidenceProvider(references); + } + + public Task GetAsync(IssueReference reference) + { + if (references.TryGetValue(reference.Canonical, out EvidenceReference? evidence)) + { + return Task.FromResult(evidence); + } + + return Task.FromResult(new EvidenceReference + { + Canonical = reference.Canonical, + Kind = reference.Kind, + Accessible = false, + State = "inaccessible", + StateReason = "inaccessible", + }); + } + + private static EvidenceReference ParseEvidence(JsonElement element, string? canonicalFromKey) + { + if (element.ValueKind != JsonValueKind.Object) + { + throw new ContractException("Each GitHub evidence entry must be an object."); + } + + HashSet allowed = ["canonical", "kind", "accessible", "state", "state_reason", "merged_at"]; + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + throw new ContractException($"Unknown GitHub evidence field '{property.Name}'."); + } + } + + string canonical = canonicalFromKey ?? RequiredString(element, "canonical"); + if (element.TryGetProperty("canonical", out JsonElement canonicalElement) && + (canonicalElement.ValueKind != JsonValueKind.String || + !string.Equals(canonicalElement.GetString(), canonical, StringComparison.Ordinal))) + { + throw new ContractException("GitHub evidence canonical key and field do not match."); + } + + string kind = RequiredString(element, "kind"); + bool accessible = true; + if (element.TryGetProperty("accessible", out JsonElement accessibleElement)) + { + if (accessibleElement.ValueKind is not (JsonValueKind.True or JsonValueKind.False)) + { + throw new ContractException("GitHub evidence accessible must be a boolean."); + } + + accessible = accessibleElement.GetBoolean(); + } + + string state = OptionalString(element, "state"); + string stateReason = OptionalNullableString(element, "state_reason") ?? ""; + string? mergedAt = OptionalNullableString(element, "merged_at"); + if (accessible && kind.Equals("issue", StringComparison.OrdinalIgnoreCase) && + state.Length == 0) + { + throw new ContractException($"Accessible issue evidence '{canonical}' requires state."); + } + + if (accessible && kind.Equals("issue", StringComparison.OrdinalIgnoreCase) && + state.Equals("closed", StringComparison.OrdinalIgnoreCase) && + stateReason.Length == 0) + { + throw new ContractException($"Closed issue evidence '{canonical}' requires state_reason."); + } + + if (accessible && NormalizeKind(kind) == "pull_request" && state.Length == 0) + { + throw new ContractException($"Accessible pull request evidence '{canonical}' requires state."); + } + + return new EvidenceReference + { + Canonical = canonical, + Kind = kind, + Accessible = accessible, + State = state, + StateReason = stateReason, + MergedAt = mergedAt, + }; + } + + private static string RequiredString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind != JsonValueKind.String) + { + throw new ContractException($"GitHub evidence {name} must be a string."); + } + + return value.GetString()!; + } + + private static string OptionalString(JsonElement element, string name) => + element.TryGetProperty(name, out JsonElement value) + ? value.ValueKind == JsonValueKind.String + ? value.GetString()! + : throw new ContractException($"GitHub evidence {name} must be a string.") + : ""; + + private static string? OptionalNullableString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind == JsonValueKind.Null) + { + return null; + } + + return value.ValueKind == JsonValueKind.String + ? value.GetString() + : throw new ContractException($"GitHub evidence {name} must be a string or null."); + } + } + + private sealed class GitHubReferenceEvidenceProvider : IReferenceEvidenceProvider + { + private readonly HttpClient _client; + + public GitHubReferenceEvidenceProvider() + { + string? token = Environment.GetEnvironmentVariable("GH_TOKEN"); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InfrastructureException("GH_TOKEN is required when --github-evidence is not supplied."); + } + + _client = new HttpClient + { + BaseAddress = new Uri("https://api.github.com/"), + Timeout = TimeSpan.FromSeconds(30), + }; + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + _client.DefaultRequestHeaders.UserAgent.ParseAdd("unskip-closed-tests-tool/1"); + _client.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github+json"); + _client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + } + + public async Task GetAsync(IssueReference reference) + { + try + { + using HttpResponseMessage issueResponse = await _client.GetAsync( + $"repos/{Uri.EscapeDataString(reference.Owner)}/{Uri.EscapeDataString(reference.Repo)}/issues/{reference.Number}"); + if (issueResponse.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) + { + if (issueResponse.Headers.TryGetValues("X-RateLimit-Remaining", out IEnumerable? remaining) && + remaining.Contains("0", StringComparer.Ordinal)) + { + throw new InfrastructureException("GitHub API rate limit was exhausted."); + } + + return Inaccessible(reference); + } + + if (issueResponse.StatusCode == HttpStatusCode.Unauthorized) + { + throw new InfrastructureException("GitHub authentication was rejected."); + } + + if (!issueResponse.IsSuccessStatusCode) + { + throw new InfrastructureException($"GitHub issue lookup returned {(int)issueResponse.StatusCode}."); + } + + using JsonDocument issue = JsonDocument.Parse(await issueResponse.Content.ReadAsStreamAsync()); + JsonElement issueRoot = issue.RootElement; + bool isPullRequest = issueRoot.TryGetProperty("pull_request", out _); + if (!isPullRequest) + { + return new EvidenceReference + { + Canonical = reference.Canonical, + Kind = "issue", + Accessible = true, + State = RequiredApiString(issueRoot, "state"), + StateReason = NullableApiString(issueRoot, "state_reason") ?? "", + }; + } + + using HttpResponseMessage pullResponse = await _client.GetAsync( + $"repos/{Uri.EscapeDataString(reference.Owner)}/{Uri.EscapeDataString(reference.Repo)}/pulls/{reference.Number}"); + if (!pullResponse.IsSuccessStatusCode) + { + if (pullResponse.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) + { + return Inaccessible(reference); + } + + throw new InfrastructureException($"GitHub pull request lookup returned {(int)pullResponse.StatusCode}."); + } + + using JsonDocument pull = JsonDocument.Parse(await pullResponse.Content.ReadAsStreamAsync()); + JsonElement pullRoot = pull.RootElement; + return new EvidenceReference + { + Canonical = reference.Canonical, + Kind = "pull_request", + Accessible = true, + State = RequiredApiString(pullRoot, "state"), + StateReason = "", + MergedAt = NullableApiString(pullRoot, "merged_at"), + }; + } + catch (InfrastructureException) + { + throw; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or IOException) + { + throw new InfrastructureException($"GitHub lookup failed for {reference.Canonical}.", ex); + } + } + + private static EvidenceReference Inaccessible(IssueReference reference) => new() + { + Canonical = reference.Canonical, + Kind = reference.Kind, + Accessible = false, + State = "inaccessible", + StateReason = "inaccessible", + }; + + private static string RequiredApiString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind != JsonValueKind.String) + { + throw new InfrastructureException($"GitHub response omitted string field '{name}'."); + } + + return value.GetString()!; + } + + private static string? NullableApiString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value)) + { + throw new InfrastructureException($"GitHub response omitted field '{name}'."); + } + + return value.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => value.GetString(), + _ => throw new InfrastructureException($"GitHub response field '{name}' is malformed."), + }; + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/JsonSupport.cs b/.github/workflows/unskip-closed-tests-tool/JsonSupport.cs new file mode 100644 index 0000000000..dce5ecf672 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/JsonSupport.cs @@ -0,0 +1,148 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace UnskipClosedTests.Tool; + +internal static class JsonSupport +{ + public static readonly JsonSerializerOptions Options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DictionaryKeyPolicy = JsonNamingPolicy.SnakeCaseLower, + WriteIndented = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never, + }; + + public static T Read(string path) + { + try + { + string json = File.ReadAllText(path, Encoding.UTF8); + return JsonSerializer.Deserialize(json, Options) + ?? throw new ContractException($"JSON document '{path}' is empty."); + } + catch (ContractException) + { + throw; + } + catch (JsonException ex) + { + throw new ContractException($"Malformed JSON in '{path}': {ex.Message}"); + } + catch (IOException ex) + { + throw new InfrastructureException($"Could not read '{path}'.", ex); + } + catch (UnauthorizedAccessException ex) + { + throw new InfrastructureException($"Could not read '{path}'.", ex); + } + } + + public static JsonDocument ReadDocument(string path) + { + try + { + return JsonDocument.Parse(File.ReadAllBytes(path)); + } + catch (JsonException ex) + { + throw new ContractException($"Malformed JSON in '{path}': {ex.Message}"); + } + catch (IOException ex) + { + throw new InfrastructureException($"Could not read '{path}'.", ex); + } + } + + public static void Write(string path, T value) + { + try + { + string fullPath = Path.GetFullPath(path); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + File.WriteAllText(fullPath, JsonSerializer.Serialize(value, Options) + Environment.NewLine, new UTF8Encoding(false)); + } + catch (IOException ex) + { + throw new InfrastructureException($"Could not write '{path}'.", ex); + } + catch (UnauthorizedAccessException ex) + { + throw new InfrastructureException($"Could not write '{path}'.", ex); + } + } + + public static string Sha256(ReadOnlySpan bytes) => + Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + + public static string Sha256(string text) => Sha256(Encoding.UTF8.GetBytes(text)); + + public static string CanonicalDigest(T value, string? excludedProperty = null) + { + JsonNode node = JsonSerializer.SerializeToNode(value, Options) + ?? throw new InfrastructureException("Could not serialize a digest input."); + if (excludedProperty is not null && node is JsonObject root) + { + root.Remove(excludedProperty); + } + + StringBuilder builder = new(); + WriteCanonical(node, builder); + return Sha256(builder.ToString()); + } + + public static string ManifestDigest(Manifest manifest) => + CanonicalDigest(manifest, "manifest_digest"); + + private static void WriteCanonical(JsonNode? node, StringBuilder builder) + { + switch (node) + { + case null: + builder.Append("null"); + break; + case JsonObject obj: + builder.Append('{'); + bool firstProperty = true; + foreach ((string key, JsonNode? value) in obj.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + if (!firstProperty) + { + builder.Append(','); + } + + firstProperty = false; + builder.Append(JsonSerializer.Serialize(key)); + builder.Append(':'); + WriteCanonical(value, builder); + } + + builder.Append('}'); + break; + case JsonArray array: + builder.Append('['); + for (int i = 0; i < array.Count; i++) + { + if (i > 0) + { + builder.Append(','); + } + + WriteCanonical(array[i], builder); + } + + builder.Append(']'); + break; + case JsonValue value: + builder.Append(value.ToJsonString()); + break; + default: + throw new InfrastructureException("Unsupported JSON node while computing a digest."); + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/ManifestValidator.cs b/.github/workflows/unskip-closed-tests-tool/ManifestValidator.cs new file mode 100644 index 0000000000..60256beacc --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/ManifestValidator.cs @@ -0,0 +1,77 @@ +namespace UnskipClosedTests.Tool; + +internal static class ManifestValidator +{ + public static Manifest Read(string path) + { + Manifest manifest = JsonSupport.Read(path); + Validate(manifest); + return manifest; + } + + public static void Validate(Manifest manifest) + { + if (manifest.SchemaVersion != "1") + { + throw new ContractException($"Unsupported manifest schema_version '{manifest.SchemaVersion}'."); + } + + if (manifest.CandidateCount != manifest.Candidates.Count) + { + throw new ContractException("candidate_count does not match candidates."); + } + + if (manifest.Repository.Split('/').Length != 2 || + manifest.SourceCommit.Length is not (40 or 64) || + manifest.GitObjectFormat is not ("sha1" or "sha256") || + manifest.ConfigDigest.Length != 64 || + manifest.ManifestDigest.Length != 64) + { + throw new ContractException("Manifest root identity fields are malformed."); + } + + if (!string.Equals(JsonSupport.ManifestDigest(manifest), manifest.ManifestDigest, StringComparison.Ordinal)) + { + throw new ContractException("manifest_digest does not match manifest content."); + } + + HashSet candidateIds = new(StringComparer.Ordinal); + foreach (Candidate candidate in manifest.Candidates) + { + if (!candidateIds.Add(candidate.CandidateId)) + { + throw new ContractException($"Duplicate candidate_id '{candidate.CandidateId}'."); + } + + PathRules.ValidateRelativePath(candidate.Path, "candidate path"); + if (!candidate.Path.EndsWith(".cs", StringComparison.OrdinalIgnoreCase) || + candidate.CandidateId.Length != 64 || + candidate.StableOwnerId.Length != 64 || + candidate.SourceSha256.Length != 64 || + candidate.AttributeTextSha256.Length != 64 || + candidate.AttributeSpan.Start < 0 || + candidate.AttributeSpan.Length <= 0 || + candidate.Owner.ContainingTypes.Count == 0 || + candidate.Owner.TypeFqn.Length == 0 || + candidate.Owner.DeclarationId.Length == 0) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has malformed trusted identity fields."); + } + + if (candidate.Owner.Kind is not ("method" or "class")) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has unsupported owner kind."); + } + + if (candidate.CanonicalIssueReferences.Count == 0) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has no concrete issue references."); + } + + if (candidate.Owner.TestFqns.Distinct(StringComparer.Ordinal).Count() != candidate.Owner.TestFqns.Count) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has duplicate test FQNs."); + } + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Models.cs b/.github/workflows/unskip-closed-tests-tool/Models.cs new file mode 100644 index 0000000000..483e2075f1 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Models.cs @@ -0,0 +1,172 @@ +using System.Text.Json.Serialization; + +namespace UnskipClosedTests.Tool; + +internal sealed class ToolConfig +{ + public string SchemaVersion { get; set; } = ""; + public List SourceRoots { get; set; } = []; + public List ExcludedGlobs { get; set; } = []; + public List GeneratedGlobs { get; set; } = []; + public List IgnoreAttributeNames { get; set; } = []; + public List TestAttributeNames { get; set; } = []; + public List VerificationCommand { get; set; } = []; + public int VerificationTimeoutSeconds { get; set; } +} + +internal sealed class Manifest +{ + public string SchemaVersion { get; set; } = "1"; + public string Repository { get; set; } = ""; + public string SourceCommit { get; set; } = ""; + public string GitObjectFormat { get; set; } = ""; + public string ConfigDigest { get; set; } = ""; + public string ManifestDigest { get; set; } = ""; + public int CandidateCount { get; set; } + public List Candidates { get; set; } = []; +} + +internal sealed class Candidate +{ + public string CandidateId { get; set; } = ""; + public string StableOwnerId { get; set; } = ""; + public string Path { get; set; } = ""; + public string BlobOid { get; set; } = ""; + public string SourceSha256 { get; set; } = ""; + public SourceSpan AttributeSpan { get; set; } = new(); + public string AttributeTextSha256 { get; set; } = ""; + public OwnerIdentity Owner { get; set; } = new(); + public List CanonicalIssueReferences { get; set; } = []; + public CandidateDecision Decision { get; set; } = new(); +} + +internal sealed class SourceSpan +{ + public int Start { get; set; } + public int Length { get; set; } + public int StartLine { get; set; } + public int StartColumn { get; set; } + public int EndLine { get; set; } + public int EndColumn { get; set; } +} + +internal sealed class OwnerIdentity +{ + public string Kind { get; set; } = ""; + public string Namespace { get; set; } = ""; + public List ContainingTypes { get; set; } = []; + public string TypeFqn { get; set; } = ""; + public string DeclarationId { get; set; } = ""; + public string MethodName { get; set; } = ""; + public string MethodSignature { get; set; } = ""; + public List TestFqns { get; set; } = []; +} + +internal sealed class IssueReference +{ + public string Kind { get; set; } = ""; + public string Owner { get; set; } = ""; + public string Repo { get; set; } = ""; + public int Number { get; set; } + public string Canonical { get; set; } = ""; + public string Url { get; set; } = ""; + public bool Eligibility { get; set; } + public string State { get; set; } = ""; + public string StateReason { get; set; } = ""; + public string? MergedAt { get; set; } +} + +internal sealed class CandidateDecision +{ + public bool Eligible { get; set; } + public List Deferrals { get; set; } = []; +} + +internal sealed class EvidenceFile +{ + public string SchemaVersion { get; set; } = ""; + public List References { get; set; } = []; +} + +internal sealed class EvidenceReference +{ + public string Canonical { get; set; } = ""; + public string Kind { get; set; } = ""; + public bool Accessible { get; set; } = true; + public string State { get; set; } = ""; + public string StateReason { get; set; } = ""; + public string? MergedAt { get; set; } +} + +internal sealed class ApplyRequest +{ + public string SchemaVersion { get; set; } = "1"; + public VerificationCandidateRequest Candidate { get; set; } = new(); + public string Repository { get; set; } = ""; + public string SourceCommit { get; set; } = ""; + public List Tests { get; set; } = []; +} + +internal sealed class VerificationCandidateRequest +{ + public string CandidateId { get; set; } = ""; +} + +internal sealed class VerificationTest +{ + public string Fqn { get; set; } = ""; + public string SourcePath { get; set; } = ""; + public string ResultFile { get; set; } = ""; +} + +internal sealed class ApplyResult +{ + public string SchemaVersion { get; set; } = "1"; + public string SourceCommit { get; set; } = ""; + public string ManifestDigest { get; set; } = ""; + public List RetainedCandidates { get; set; } = []; + public List RevertedCandidates { get; set; } = []; + public List ChangedPaths { get; set; } = []; + public bool HasChanges { get; set; } + public string PrTitle { get; set; } = ""; + public string PrBody { get; set; } = ""; +} + +internal sealed class RetainedCandidateResult +{ + public string CandidateId { get; set; } = ""; + public string Path { get; set; } = ""; + public List TestFqns { get; set; } = []; +} + +internal sealed class RevertedCandidateResult +{ + public string CandidateId { get; set; } = ""; + public string Path { get; set; } = ""; + public List TestFqns { get; set; } = []; + public string Reason { get; set; } = ""; +} + +internal sealed class CliOptions +{ + public string Command { get; init; } = ""; + public Dictionary Values { get; init; } = new(StringComparer.Ordinal); + + public string Required(string name) => + Values.TryGetValue(name, out string? value) && value.Length > 0 + ? value + : throw new ContractException($"Missing required option --{name}."); + + public string? Optional(string name) => Values.GetValueOrDefault(name); +} + +internal sealed class ContractException(string message) : Exception(message); +internal sealed class InfrastructureException(string message, Exception? inner = null) : Exception(message, inner); + +internal static class ExitCodes +{ + public const int Success = 0; + public const int CleanNoOp = 10; + public const int Invalid = 20; + public const int Infrastructure = 30; +} diff --git a/.github/workflows/unskip-closed-tests-tool/PathRules.cs b/.github/workflows/unskip-closed-tests-tool/PathRules.cs new file mode 100644 index 0000000000..2edaadd0ad --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/PathRules.cs @@ -0,0 +1,72 @@ +using System.Text.RegularExpressions; + +namespace UnskipClosedTests.Tool; + +internal static class PathRules +{ + public static string ValidateRelativePath(string value, string context) + { + if (string.IsNullOrWhiteSpace(value) || Path.IsPathRooted(value)) + { + throw new ContractException($"{context} '{value}' must be a non-empty repository-relative path."); + } + + string normalized = value.Replace('\\', '/'); + string[] parts = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0 || parts.Any(static part => part is "." or "..")) + { + throw new ContractException($"{context} '{value}' contains traversal or an empty path."); + } + + return string.Join('/', parts); + } + + public static string ResolveInsideRoot(string repoRoot, string relativePath, string context) + { + string normalized = ValidateRelativePath(relativePath, context); + string root = Path.GetFullPath(repoRoot); + string fullPath = Path.GetFullPath(Path.Combine(root, normalized.Replace('/', Path.DirectorySeparatorChar))); + string prefix = root.EndsWith(Path.DirectorySeparatorChar) ? root : root + Path.DirectorySeparatorChar; + if (!fullPath.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException($"{context} '{relativePath}' escapes the repository root."); + } + + return fullPath; + } + + public static void RejectReparsePoints(string repoRoot, string fullPath) + { + string root = Path.GetFullPath(repoRoot).TrimEnd(Path.DirectorySeparatorChar); + string current = Path.GetFullPath(fullPath); + while (!string.Equals(current, root, StringComparison.OrdinalIgnoreCase)) + { + if (!File.Exists(current) && !Directory.Exists(current)) + { + throw new ContractException($"Source path '{fullPath}' does not exist."); + } + + if ((File.GetAttributes(current) & FileAttributes.ReparsePoint) != 0) + { + throw new ContractException($"Source path '{fullPath}' traverses a symlink or reparse point."); + } + + current = Path.GetDirectoryName(current) + ?? throw new ContractException($"Source path '{fullPath}' is outside the repository."); + } + } + + public static bool MatchesAnyGlob(string path, IEnumerable globs) => + globs.Any(glob => GlobToRegex(glob).IsMatch(path)); + + private static Regex GlobToRegex(string glob) + { + string normalized = glob.Replace('\\', '/'); + string pattern = Regex.Escape(normalized) + .Replace(@"\*\*/", "(?:.*/)?", StringComparison.Ordinal) + .Replace(@"\*\*", ".*", StringComparison.Ordinal) + .Replace(@"\*", "[^/]*", StringComparison.Ordinal) + .Replace(@"\?", "[^/]", StringComparison.Ordinal); + return new Regex($"^{pattern}$", RegexOptions.CultureInvariant | RegexOptions.NonBacktracking); + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Program.cs b/.github/workflows/unskip-closed-tests-tool/Program.cs new file mode 100644 index 0000000000..6ca86ecbf2 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Program.cs @@ -0,0 +1,149 @@ +namespace UnskipClosedTests.Tool; + +internal static class Program +{ + private const string HelpText = + """ + Usage: + UnskipClosedTests.Tool inventory --config --output [--repo-root ] [--repository ] [--source-commit ] + UnskipClosedTests.Tool resolve --manifest --output [--github-evidence ] + UnskipClosedTests.Tool apply --config --manifest --agent-output --output [--repo-root ] [--github-evidence ] + + Exit codes: + 0 Successful inventory/resolve, or apply retained at least one verified edit. + 10 Apply retained no verified candidates and left candidate source files unchanged. + 20 Invalid or stale trusted input. + 30 Infrastructure or verification protocol failure. + """; + + public static async Task Main(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine(HelpText); + return ExitCodes.Invalid; + } + + if (args[0] is "--help" or "-h" or "help" || + args.Length == 2 && args[1] is "--help" or "-h") + { + Console.WriteLine(HelpText); + return ExitCodes.Success; + } + + try + { + CliOptions options = Parse(args); + return options.Command switch + { + "inventory" => RunInventory(options), + "resolve" => await RunResolveAsync(options), + "apply" => await RunApplyAsync(options), + _ => throw new ContractException($"Unknown command '{options.Command}'. Expected inventory, resolve, or apply."), + }; + } + catch (ContractException ex) + { + Console.Error.WriteLine($"invalid: {ex.Message}"); + return ExitCodes.Invalid; + } + catch (InfrastructureException ex) + { + Console.Error.WriteLine($"infrastructure: {ex.Message}"); + return ExitCodes.Infrastructure; + } + catch (Exception ex) + { + Console.Error.WriteLine($"infrastructure: unexpected failure: {ex}"); + return ExitCodes.Infrastructure; + } + } + + private static int RunInventory(CliOptions options) + { + string configPath = options.Required("config"); + string outputPath = options.Required("output"); + string repoRoot = options.Optional("repo-root") ?? Environment.CurrentDirectory; + ToolConfig config = ConfigLoader.Load(configPath); + Manifest manifest = InventoryEngine.Create(repoRoot, options.Optional("repository"), config); + string? expectedSourceCommit = options.Optional("source-commit"); + if (expectedSourceCommit is not null && + !string.Equals(expectedSourceCommit, manifest.SourceCommit, StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException( + $"Expected source commit '{expectedSourceCommit}' does not match checked-out commit '{manifest.SourceCommit}'."); + } + + JsonSupport.Write(outputPath, manifest); + return ExitCodes.Success; + } + + private static async Task RunResolveAsync(CliOptions options) + { + Manifest manifest = ManifestValidator.Read(options.Required("manifest")); + string outputPath = options.Required("output"); + Manifest resolved = await IssueResolver.ResolveAsync(manifest, options.Optional("github-evidence")); + JsonSupport.Write(outputPath, resolved); + return ExitCodes.Success; + } + + private static async Task RunApplyAsync(CliOptions options) + { + string configPath = options.Required("config"); + ToolConfig config = ConfigLoader.Load(configPath); + ApplyResult result = await ApplyEngine.ApplyAsync( + options.Optional("repo-root") ?? Environment.CurrentDirectory, + config, + options.Required("manifest"), + options.Required("agent-output"), + options.Optional("github-evidence")); + JsonSupport.Write(options.Required("output"), result); + return result.HasChanges ? ExitCodes.Success : ExitCodes.CleanNoOp; + } + + private static CliOptions Parse(string[] args) + { + if (args.Length == 0) + { + throw new ContractException("A command is required."); + } + + Dictionary values = new(StringComparer.Ordinal); + for (int i = 1; i < args.Length; i++) + { + string item = args[i]; + if (!item.StartsWith("--", StringComparison.Ordinal) || item.Length == 2) + { + throw new ContractException($"Unexpected argument '{item}'."); + } + + string name = item[2..]; + if (i + 1 >= args.Length || args[i + 1].StartsWith("--", StringComparison.Ordinal)) + { + throw new ContractException($"Option --{name} requires a value."); + } + + if (!values.TryAdd(name, args[++i])) + { + throw new ContractException($"Option --{name} was specified more than once."); + } + } + + HashSet allowed = args[0] switch + { + "inventory" => ["config", "output", "repo-root", "repository", "source-commit"], + "resolve" => ["manifest", "output", "github-evidence"], + "apply" => ["config", "manifest", "agent-output", "output", "repo-root", "github-evidence"], + _ => [], + }; + foreach (string name in values.Keys) + { + if (!allowed.Contains(name)) + { + throw new ContractException($"Option --{name} is not valid for command '{args[0]}'."); + } + } + + return new CliOptions { Command = args[0], Values = values }; + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/TrxVerifier.cs b/.github/workflows/unskip-closed-tests-tool/TrxVerifier.cs new file mode 100644 index 0000000000..93d2c84fb2 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/TrxVerifier.cs @@ -0,0 +1,117 @@ +using System.Xml; +using System.Xml.Linq; + +namespace UnskipClosedTests.Tool; + +internal static class TrxVerifier +{ + public static (bool Success, string Reason) Verify(IReadOnlyList tests) + { + HashSet expectedFiles = tests + .Select(static test => Path.GetFullPath(test.ResultFile)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (string directory in tests.Select(static test => Path.GetDirectoryName(test.ResultFile)!) + .Distinct(StringComparer.OrdinalIgnoreCase)) + { + if (Directory.Exists(directory)) + { + string? unexpected = Directory.EnumerateFiles(directory, "*.trx", SearchOption.TopDirectoryOnly) + .Select(Path.GetFullPath) + .FirstOrDefault(path => !expectedFiles.Contains(path)); + if (unexpected is not null) + { + return (false, $"unexpected_trx:{Path.GetFileName(unexpected)}"); + } + } + } + + foreach (VerificationTest test in tests) + { + if (!File.Exists(test.ResultFile)) + { + return (false, $"missing_trx:{Path.GetFileName(test.ResultFile)}"); + } + + try + { + XDocument document = XDocument.Load(test.ResultFile, LoadOptions.None); + Dictionary mappings = new(StringComparer.Ordinal); + foreach (XElement unitTest in document.Descendants().Where(static element => + element.Name.LocalName == "UnitTest")) + { + string? id = unitTest.Attribute("id")?.Value; + XElement? method = unitTest.Descendants().FirstOrDefault(static element => + element.Name.LocalName == "TestMethod"); + string? className = method?.Attribute("className")?.Value; + string? methodName = method?.Attribute("name")?.Value; + if (string.IsNullOrWhiteSpace(id) || + string.IsNullOrWhiteSpace(className) || + string.IsNullOrWhiteSpace(methodName)) + { + return (false, "malformed_trx_test_definition"); + } + + string fqn = $"{className}.{methodName}"; + if (!string.Equals(fqn, test.Fqn, StringComparison.Ordinal)) + { + return (false, $"mismatched_fqn:{fqn}"); + } + + if (!mappings.TryAdd(id, fqn)) + { + return (false, $"duplicate_trx_test_id:{id}"); + } + } + + if (mappings.Count == 0) + { + return (false, "zero_selected_tests"); + } + + List results = document.Descendants().Where(static element => + element.Name.LocalName == "UnitTestResult").ToList(); + if (results.Count == 0) + { + return (false, "zero_executed_tests"); + } + + int passed = 0; + foreach (XElement result in results) + { + string? testId = result.Attribute("testId")?.Value; + string? outcome = result.Attribute("outcome")?.Value; + if (string.IsNullOrWhiteSpace(testId) || + !mappings.TryGetValue(testId, out string? mappedFqn)) + { + return (false, "result_without_exact_test_mapping"); + } + + if (!string.Equals(mappedFqn, test.Fqn, StringComparison.Ordinal)) + { + return (false, $"mismatched_result_fqn:{mappedFqn}"); + } + + if (string.Equals(outcome, "Passed", StringComparison.OrdinalIgnoreCase)) + { + passed++; + } + else + { + return (false, $"non_passing_outcome:{outcome ?? "missing"}"); + } + } + + if (passed == 0) + { + return (false, "no_executed_pass"); + } + } + catch (Exception ex) when (ex is XmlException or IOException or UnauthorizedAccessException) + { + return (false, $"malformed_trx:{ex.GetType().Name}"); + } + } + + return (true, "passed"); + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj b/.github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj new file mode 100644 index 0000000000..6a619d28ee --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj @@ -0,0 +1,19 @@ + + + Exe + net8.0 + enable + enable + true + true + true + + false + false + false + + + + + + diff --git a/.github/workflows/unskip-closed-tests-tool/global.json b/.github/workflows/unskip-closed-tests-tool/global.json new file mode 100644 index 0000000000..4c4c3ae5ed --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.100", + "rollForward": "latestFeature", + "allowPrerelease": false + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/packages.lock.json b/.github/workflows/unskip-closed-tests-tool/packages.lock.json new file mode 100644 index 0000000000..0c4bdcbb56 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/packages.lock.json @@ -0,0 +1,47 @@ +{ + "version": 1, + "dependencies": { + "net8.0": { + "Microsoft.CodeAnalysis.CSharp": { + "type": "Direct", + "requested": "[4.14.0, )", + "resolved": "4.14.0", + "contentHash": "568a6wcTivauIhbeWcCwfWwIn7UV7MeHEBvFB2uzGIpM2OhJ4eM/FZ8KS0yhPoNxnSpjGzz7x7CIjTxhslojQA==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "Microsoft.CodeAnalysis.Common": "[4.14.0]", + "System.Collections.Immutable": "9.0.0", + "System.Reflection.Metadata": "9.0.0" + } + }, + "Microsoft.CodeAnalysis.Analyzers": { + "type": "Transitive", + "resolved": "3.11.0", + "contentHash": "v/EW3UE8/lbEYHoC2Qq7AR/DnmvpgdtAMndfQNmpuIMx/Mto8L5JnuCfdBYtgvalQOtfNCnxFejxuRrryvUTsg==" + }, + "Microsoft.CodeAnalysis.Common": { + "type": "Transitive", + "resolved": "4.14.0", + "contentHash": "PC3tuwZYnC+idaPuoC/AZpEdwrtX7qFpmnrfQkgobGIWiYmGi5MCRtl5mx6QrfMGQpK78X2lfIEoZDLg/qnuHg==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "System.Collections.Immutable": "9.0.0", + "System.Reflection.Metadata": "9.0.0" + } + }, + "System.Collections.Immutable": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "QhkXUl2gNrQtvPmtBTQHb0YsUrDiDQ2QS09YbtTTiSjGcf7NBqtYbrG/BE06zcBPCKEwQGzIv13IVdXNOSub2w==" + }, + "System.Reflection.Metadata": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "ANiqLu3DxW9kol/hMmTWbt3414t9ftdIuiIU7j80okq2YzAueo120M442xk1kDJWtmZTqWQn7wHDvMRipVOEOQ==", + "dependencies": { + "System.Collections.Immutable": "9.0.0" + } + } + } + } +} \ No newline at end of file diff --git a/.github/workflows/unskip-closed-tests-verify.sh b/.github/workflows/unskip-closed-tests-verify.sh new file mode 100644 index 0000000000..8ecba422c7 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-verify.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "The repository must replace verification.command with its trusted build/test hook." >&2 +exit 2 diff --git a/.github/workflows/unskip-closed-tests.config.json b/.github/workflows/unskip-closed-tests.config.json new file mode 100644 index 0000000000..6ffaea8b5d --- /dev/null +++ b/.github/workflows/unskip-closed-tests.config.json @@ -0,0 +1,40 @@ +{ + "schema_version": "1", + "source_roots": [ + "test", + "samples" + ], + "excluded_globs": [ + "**/bin/**", + "**/obj/**" + ], + "generated_globs": [ + "**/Generated/**", + "**/generated/**", + "**/*.Designer.cs", + "**/*.g.cs", + "**/*.generated.cs" + ], + "ignore_attribute_names": [ + "Ignore", + "IgnoreAttribute" + ], + "test_attribute_names": [ + "TestMethod", + "TestMethodAttribute", + "DataTestMethod", + "DataTestMethodAttribute", + "STATestMethod", + "STATestMethodAttribute", + "UITestMethod", + "UITestMethodAttribute" + ], + "verification": { + "command": [ + "python3", + ".github/scripts/unskip_closed_tests_verify.py", + "{request_json}" + ], + "timeout_seconds": 1800 + } +} diff --git a/.github/workflows/unskip-closed-tests.lock.yml b/.github/workflows/unskip-closed-tests.lock.yml index 06669062e1..23ced58630 100644 --- a/.github/workflows/unskip-closed-tests.lock.yml +++ b/.github/workflows/unskip-closed-tests.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a2670f95b69128450276ecb60b5278111ad9dcff9b0a63a51266121d511f1575","body_hash":"55b2626bdc15c282ff69e139f600d4d1d317542ca927a24385ae39bc6da98a31","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","detection_agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} -# gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.12.2","digest":"sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6","pinned_image":"ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_pull_request","missing_data","missing_tool","noop"]}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c3d12facfb6e181b643684c6c47f7716b1ee1a02657bac7585d86f307d580b7a","body_hash":"8f1de6229aed153de2ce0e740ba73fdeee568b085232b4c017e5d305c134f3de","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} +# gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-dotnet","sha":"a98b56852c35b8e3190ac28c8c2271da59106c68","version":"v6.0.0 (source v6)"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"}],"mcp_servers":[{"name":"safeoutputs","tools":["apply_verified_unskips","missing_data","missing_tool","noop"]}]} # This file was automatically generated by gh-aw (v0.89.21). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -17,20 +17,22 @@ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # # -# To update this file, edit the corresponding .md file and run: +# To update this file, edit dotnet/skills/agentic-workflows/unskip-closed-tests@7bdab53812ed1ce4fe2cb6b0cf84e17c8ff0f097 and run: # gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Scans the test suite for skipped/ignored tests whose linked tracking issue is already closed, then opens a pull request that re-enables (unskips) them. +# Inventories source-bound .NET Ignore attributes at one trusted revision, permits a read-only agent to select only verified sites, and opens at most one draft pull request after deterministic issue and test-result validation. +# +# Source: dotnet/skills/agentic-workflows/unskip-closed-tests@7bdab53812ed1ce4fe2cb6b0cf84e17c8ff0f097 # # Resolved workflow manifest: # Imports: -# - shared/repo-build-setup.md +# - unskip-closed-tests-prepare.md +# - unskip-closed-tests-shared.md # # Secrets used: -# - GH_AW_CI_TRIGGER_TOKEN # - GH_AW_DEFAULT_OTLP_ENDPOINT # - GH_AW_DEFAULT_OTLP_HEADERS # - GH_AW_GITHUB_MCP_SERVER_TOKEN @@ -39,11 +41,13 @@ # # Custom actions used: # - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 -# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 (source v6) # - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) # - github/gh-aw-actions/setup@924af5fdc64061cfbf66fb584c8b07e2ac230c60 # v0.89.21 # # Container images used: @@ -52,13 +56,12 @@ # - ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0 # - ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086 # - ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 -# - ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6 name: "Unskip Closed Tests" on: schedule: - cron: "50 20 * * 0" # Friendly format: weekly (scattered) - # skip-if-match: is:pr is:open in:title "[unskip-tests]" # Skip-if-match processed as search check in pre-activation job + # skip-if-match: is:pr is:open in:title "[unskip-closed-tests]" # Skip-if-match processed as search check in pre-activation job workflow_dispatch: inputs: aw_context: @@ -125,6 +128,7 @@ jobs: GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/unskip-closed-tests.lock.yml@${{ github.ref }} GH_AW_INFO_VERSION: "1.0.87" GH_AW_INFO_AWF_VERSION: "v0.28.23" + GH_AW_INFO_BODY_MODIFIED: "false" GH_AW_INFO_ENGINE_ID: "copilot" - name: Mask OTLP telemetry headers run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" @@ -147,6 +151,8 @@ jobs: GH_AW_INFO_AWMG_VERSION: "" GH_AW_INFO_FIREWALL_TYPE: "squid" GH_AW_INFO_AGENT_RUNTIME: "" + GH_AW_INFO_FRONTMATTER_SOURCE: "dotnet/skills/agentic-workflows/unskip-closed-tests@7bdab53812ed1ce4fe2cb6b0cf84e17c8ff0f097" + GH_AW_INFO_BODY_MODIFIED: "false" GH_AW_COMPILED_STRICT: "true" uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -268,21 +274,13 @@ jobs: GH_AW_ACTIONS_DIR: ${{ runner.temp }}/gh-aw/actions GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl - GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"file\":\"safe_outputs_create_pull_request.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0006\"}]}" - GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0006\"}]}" GH_AW_PROMPT_CONTENT_0000: "\n" - GH_AW_PROMPT_CONTENT_0001: "\nTools: create_pull_request, missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: missing_tool, missing_data, noop, apply_verified_unskips\n" GH_AW_PROMPT_CONTENT_0002: "\n" - GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" - GH_AW_PROMPT_CONTENT_0004: "\n" - GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/shared/repo-build-setup.md}}\n" + GH_AW_PROMPT_CONTENT_0003: "\n" + GH_AW_PROMPT_CONTENT_0004: "{{#runtime-import .github/workflows/unskip-closed-tests-prepare.md}}\n" + GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/unskip-closed-tests-shared.md}}\n" GH_AW_PROMPT_CONTENT_0006: "{{#runtime-import .github/workflows/unskip-closed-tests.md}}\n" with: script: | @@ -295,8 +293,6 @@ jobs: env: GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} with: script: | const path = require('path'); @@ -309,15 +305,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt - GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_MCP_CLI_SERVERS_LIST: "- `github` — run `github --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} with: script: | @@ -332,14 +320,6 @@ jobs: return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, substitutions: { - GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, - GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, - GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, - GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED } @@ -385,8 +365,11 @@ jobs: retention-days: 1 agent: - needs: activation - if: needs.activation.outputs.daily_ai_credits_exceeded != 'true' + needs: + - activation + - collect-unskip-candidates + if: > + (needs.activation.outputs.daily_ai_credits_exceeded != 'true') && (needs.collect-unskip-candidates.outputs.eligible-count != '0') runs-on: ubuntu-latest permissions: contents: read @@ -448,6 +431,7 @@ jobs: GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/unskip-closed-tests.lock.yml@${{ github.ref }} GH_AW_INFO_VERSION: "1.0.87" GH_AW_INFO_AWF_VERSION: "v0.28.23" + GH_AW_INFO_BODY_MODIFIED: "false" GH_AW_INFO_ENGINE_ID: "copilot" - name: Set runtime paths id: set-runtime-paths @@ -487,10 +471,18 @@ jobs: with: name: activation path: /tmp/gh-aw - - name: Build - run: ./build.sh - - name: Put dotnet on the path - run: echo "$PWD/.dotnet" >> $GITHUB_PATH + - name: Download trusted candidate manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: .gh-aw/unskip-closed-tests + - env: + GH_AW_MANIFEST_DIGEST_VALUE: ${{ needs.collect-unskip-candidates.outputs.manifest-digest }} + GH_AW_SOURCE_COMMIT_VALUE: ${{ needs.collect-unskip-candidates.outputs.source-commit }} + GH_AW_WORKSPACE_VALUE: ${{ github.workspace }} + name: Export trusted manifest context + run: "{\n echo \"GH_AW_UNSKIP_MANIFEST=${GH_AW_WORKSPACE_VALUE}/.gh-aw/unskip-closed-tests/manifest.json\"\n echo \"GH_AW_UNSKIP_SOURCE_COMMIT=${GH_AW_SOURCE_COMMIT_VALUE}\"\n echo \"GH_AW_UNSKIP_MANIFEST_DIGEST=${GH_AW_MANIFEST_DIGEST_VALUE}\"\n} >> \"$GITHUB_ENV\"\n" + shell: bash - name: Configure Git credentials env: @@ -521,18 +513,6 @@ jobs: GH_AW_COMPILED_VERSION: v0.89.21 - name: Install AWF binary run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.23 --rootless - - name: Determine automatic lockdown mode for GitHub MCP Server - id: determine-automatic-lockdown - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) - env: - GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} - GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} - with: - script: | - const path = require('path'); - const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); - const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); - await determineAutomaticLockdown(github, context, core); - name: Restore agent config folders from base branch if: steps.checkout-pr.outcome == 'success' env: @@ -549,7 +529,7 @@ jobs: GH_AW_SKILL_DIR: ".github/skills" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64 ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0 ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086 ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6 + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64 ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0 ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086 ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 - name: Prepare Safe Outputs Directories run: | mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" @@ -560,7 +540,7 @@ jobs: env: GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" - GH_AW_SAFE_OUTPUTS_CONFIG: "{\"create_pull_request\":{\"draft\":true,\"expires\":48,\"labels\":[\"type/automation\",\"type/test-gap\"],\"max\":1,\"max_patch_files\":100,\"max_patch_size\":4096,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\"],\"protected_files_policy\":\"fallback-to-issue\",\"title_prefix\":\"[unskip-tests] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"apply-verified-unskips\":{\"description\":\"Revalidate selected source sites and tracking items, apply only trusted Ignore removals, require exact structured test execution evidence, and open at most one draft pull request.\",\"inputs\":{\"candidate_ids_json\":{\"default\":null,\"description\":\"JSON array of exact candidate IDs copied from the manifest.\",\"required\":true,\"type\":\"string\"},\"manifest_digest\":{\"default\":null,\"description\":\"Exact trusted manifest digest.\",\"required\":true,\"type\":\"string\"}}},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" with: script: | const path = require('path'); @@ -573,73 +553,35 @@ jobs: env: GH_AW_TOOLS_META_JSON: | { - "description_suffixes": { - "create_pull_request": " CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"[unskip-tests] \". Labels [\"type/automation\" \"type/test-gap\"] will be automatically added. PRs will be created as drafts." - }, + "description_suffixes": {}, "repo_params": {}, - "dynamic_tools": [] + "dynamic_tools": [ + { + "description": "Revalidate selected source sites and tracking items, apply only trusted Ignore removals, require exact structured test execution evidence, and open at most one draft pull request.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "candidate_ids_json": { + "description": "JSON array of exact candidate IDs copied from the manifest.", + "type": "string" + }, + "manifest_digest": { + "description": "Exact trusted manifest digest.", + "type": "string" + } + }, + "required": [ + "candidate_ids_json", + "manifest_digest" + ], + "type": "object" + }, + "name": "apply_verified_unskips" + } + ] } GH_AW_VALIDATION_JSON: | { - "create_pull_request": { - "defaultMax": 1, - "fields": { - "base": { - "type": "string", - "sanitize": true, - "maxLength": 128 - }, - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "branch": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "dependencies": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 256 - }, - "draft": { - "type": "boolean" - }, - "labels": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 - }, - "repo": { - "type": "string", - "maxLength": 256 - }, - "stack_position": { - "optionalPositiveInteger": true - }, - "stack_root": { - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "temporary_id": { - "type": "string", - "pattern": "^#?aw_[A-Za-z0-9_]{3,12}$" - }, - "title": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, "missing_data": { "defaultMax": 20, "fields": { @@ -730,10 +672,6 @@ jobs: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} - GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} - GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} - GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eo pipefail @@ -771,26 +709,9 @@ jobs: mkdir -p "$HOME/.copilot" GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_6e47705499a41fbb_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_763bc2030deabb7d_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { - "github": { - "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v1.12.2", - "env": { - "GITHUB_FEATURES": "fields_param", - "GITHUB_HOST": "${GITHUB_SERVER_URL}", - "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}", - "GITHUB_READ_ONLY": "1", - "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" - }, - "guard-policies": { - "allow-only": { - "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", - "repos": "$GITHUB_MCP_GUARD_REPOS" - } - } - }, "safeoutputs": { "type": "stdio", "container": "ghcr.io/github/gh-aw-node", @@ -822,14 +743,6 @@ jobs: "GITHUB_TOKEN": "\${GITHUB_TOKEN}", "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}", "RUNNER_TEMP": "\${RUNNER_TEMP}" - }, - "guard-policies": { - "write-sink": { - "accept": [ - "*" - ], - "sink-visibility": "${GH_AW_SINK_VISIBILITY}" - } } } }, @@ -846,7 +759,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_6e47705499a41fbb_EOF + GH_AW_MCP_CONFIG_763bc2030deabb7d_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -873,37 +786,24 @@ jobs: - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool github # --allow-tool safeoutputs # --allow-tool shell(cat) # --allow-tool shell(date) - # --allow-tool shell(dotnet:*) # --allow-tool shell(echo) - # --allow-tool shell(find) - # --allow-tool shell(git add:*) - # --allow-tool shell(git branch:*) - # --allow-tool shell(git checkout:*) - # --allow-tool shell(git commit:*) - # --allow-tool shell(git merge:*) - # --allow-tool shell(git rm:*) - # --allow-tool shell(git status) - # --allow-tool shell(git switch:*) - # --allow-tool shell(git:*) - # --allow-tool shell(github:*) # --allow-tool shell(grep) # --allow-tool shell(head) + # --allow-tool shell(jq) # --allow-tool shell(ls) # --allow-tool shell(printf) # --allow-tool shell(pwd) - # --allow-tool shell(rg) # --allow-tool shell(safeoutputs:*) + # --allow-tool shell(sed) # --allow-tool shell(sort) # --allow-tool shell(tail) # --allow-tool shell(uniq) # --allow-tool shell(wc) # --allow-tool shell(yq) - # --allow-tool write - timeout-minutes: 30 + timeout-minutes: 20 run: | set -o pipefail printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt @@ -962,8 +862,8 @@ jobs: GH_AW_AWF_LOG_FILE=/tmp/gh-aw/agent-stdio.log \ GH_AW_AWF_ATTEMPT_LOG_NAME=copilot \ bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ - awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; GH_AW_TOOL_BINS="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')"; GH_AW_TOOL_BINS="${GH_AW_TOOL_BINS%:}"; export PATH="$PATH${GH_AW_TOOL_BINS:+:}$GH_AW_TOOL_BINS"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(dotnet:*)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(git add:*)'\'' --allow-tool '\''shell(git branch:*)'\'' --allow-tool '\''shell(git checkout:*)'\'' --allow-tool '\''shell(git commit:*)'\'' --allow-tool '\''shell(git merge:*)'\'' --allow-tool '\''shell(git rm:*)'\'' --allow-tool '\''shell(git status)'\'' --allow-tool '\''shell(git switch:*)'\'' --allow-tool '\''shell(git:*)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(rg)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; GH_AW_TOOL_BINS="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')"; GH_AW_TOOL_BINS="${GH_AW_TOOL_BINS%:}"; export PATH="$PATH${GH_AW_TOOL_BINS:+:}$GH_AW_TOOL_BINS"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sed)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -976,13 +876,12 @@ jobs: GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_TIMEOUT_MINUTES: 30 + GH_AW_TIMEOUT_MINUTES: 20 GH_AW_VERSION: v0.89.21 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md @@ -1000,7 +899,7 @@ jobs: continue-on-error: true env: GH_AW_AGENTIC_EXECUTION_OUTCOME: ${{ steps.agentic_execution.outcome }} - GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: 30 + GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: 20 uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | @@ -1193,10 +1092,283 @@ jobs: /tmp/gh-aw/sandbox/firewall/awf-reflect.json if-no-files-found: ignore + apply_verified_unskips: + needs: + - agent + - detection + if: > + (!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'apply_verified_unskips') && + (needs.agent.result == 'success' && needs.detection.result == 'success' && needs.detection.outputs.detection_success == 'true' && + contains(needs.agent.outputs.output_types, 'apply_verified_unskips')) + runs-on: ubuntu-latest + permissions: + contents: write + issues: read + pull-requests: write + steps: + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: ${{ runner.temp }}/gh-aw/safe-jobs/ + - name: Checkout exact analyzed revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + with: + fetch-depth: 0.0 + persist-credentials: true + ref: ${{ github.sha }} + - name: Set up .NET SDK + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 (source v6) + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + with: + dotnet-version: 8.0.x + - name: Restore trusted apply tool + run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + working-directory: .github/workflows/unskip-closed-tests-tool + - name: Download original trusted manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + - name: Revalidate, edit, and verify selected candidates + id: apply + run: | + set -euo pipefail + set +e + dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- apply \ + --repo-root "$GITHUB_WORKSPACE" \ + --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ + --manifest "$ORIGINAL_MANIFEST" \ + --agent-output "$GH_AW_AGENT_OUTPUT" \ + --output "$RESULT_PATH" + APPLY_EXIT=$? + rm -rf bin obj + set -e + + if [ "$APPLY_EXIT" -eq 10 ]; then + git diff --quiet + echo "no-action=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ "$APPLY_EXIT" -ne 0 ]; then + exit "$APPLY_EXIT" + fi + + test -f "$RESULT_PATH" + jq -e \ + '.schema_version == "1" and .has_changes == true and (.changed_paths | length > 0)' \ + "$RESULT_PATH" >/dev/null + echo "no-action=false" >> "$GITHUB_OUTPUT" + echo "result-path=$RESULT_PATH" >> "$GITHUB_OUTPUT" + env: + EXPECTED_COMMIT: ${{ github.sha }} + EXPECTED_REPOSITORY: ${{ github.repository }} + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + GH_TOKEN: ${{ github.token }} + ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json + RESULT_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-results + RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-result.json + shell: bash + working-directory: .github/workflows/unskip-closed-tests-tool + - name: Publish one verified draft pull request + if: steps.apply.outputs.no-action == 'false' + run: | + set -euo pipefail + + DEFAULT_BRANCH=$(gh api "repos/${EXPECTED_REPOSITORY}" --jq '.default_branch') + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + test "$CURRENT_HEAD" = "$EXPECTED_COMMIT" || + { echo "::notice::Default branch advanced; leaving verified changes unpublished."; exit 0; } + + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + test "$EXISTING" -eq 0 || + { echo "::notice::An unskip pull request is already open."; exit 0; } + + TITLE=$(jq -r '.pr_title' "$RESULT_PATH") + BODY_FILE="$RUNNER_TEMP/unskip-closed-tests-pr-body.md" + EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.txt" + ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.txt" + jq -r '.pr_body' "$RESULT_PATH" > "$BODY_FILE" + jq -r '.changed_paths[]' "$RESULT_PATH" | sort > "$EXPECTED_PATHS" + git diff --name-only --diff-filter=M | sort > "$ACTUAL_PATHS" + diff -u "$EXPECTED_PATHS" "$ACTUAL_PATHS" + while IFS= read -r path; do + test -n "$path" + test "${path#/}" = "$path" + case "/$path/" in + *"/../"*|*"/./"*) exit 20 ;; + esac + case "$path" in + *.cs) ;; + *) echo "::error::Unexpected changed path: $path"; exit 20 ;; + esac + git add -- "$path" + done < "$EXPECTED_PATHS" + + test -n "$(git diff --cached --name-only)" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Re-enable tests with resolved tracking items" + + BRANCH="automation/unskip-closed-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + git push origin "HEAD:refs/heads/$BRANCH" + + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + if [ "$CURRENT_HEAD" != "$EXPECTED_COMMIT" ]; then + git push origin --delete "$BRANCH" + echo "::notice::Default branch advanced before PR creation; removed the unpublished branch." + exit 0 + fi + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + if [ "$EXISTING" -ne 0 ]; then + git push origin --delete "$BRANCH" + echo "::notice::Another unskip pull request opened; removed the duplicate branch." + exit 0 + fi + + PR_URL=$(gh pr create \ + --repo "$EXPECTED_REPOSITORY" \ + --base "$DEFAULT_BRANCH" \ + --head "$BRANCH" \ + --draft \ + --title "$TITLE" \ + --body-file "$BODY_FILE") + + LIVE=$(gh pr view "$PR_URL" \ + --repo "$EXPECTED_REPOSITORY" \ + --json body,isDraft,headRefName,baseRefName,url) + test "$(printf '%s' "$LIVE" | jq -r '.isDraft')" = "true" + test "$(printf '%s' "$LIVE" | jq -r '.headRefName')" = "$BRANCH" + test "$(printf '%s' "$LIVE" | jq -r '.baseRefName')" = "$DEFAULT_BRANCH" + printf '%s' "$LIVE" | jq -r '.body' | grep -F '")) + else + (.retained_candidates | length) == 0 and + (.changed_files | length) == 0 and + (.changed_paths | length) == 0 and + .pr_title == "" and + .pr_body == "" + end' \ "$RESULT_PATH" >/dev/null - echo "no-action=false" >> "$GITHUB_OUTPUT" - echo "result-path=$RESULT_PATH" >> "$GITHUB_OUTPUT" - - name: Publish one verified draft pull request - if: steps.apply.outputs.no-action == 'false' - shell: bash - env: - GH_TOKEN: ${{ github.token }} - EXPECTED_REPOSITORY: ${{ github.repository }} - EXPECTED_COMMIT: ${{ github.sha }} - RESULT_PATH: ${{ steps.apply.outputs.result-path }} - run: | - set -euo pipefail + if [ "$(jq -r '.has_changes' "$RESULT_PATH")" = "false" ]; then + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -print -quit)" + git diff --quiet + echo "::notice::No selected candidate passed verification." + exit 0 + fi DEFAULT_BRANCH=$(gh api "repos/${EXPECTED_REPOSITORY}" --jq '.default_branch') CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') @@ -220,14 +374,20 @@ safe-outputs: EXPECTED_FILES="$RUNNER_TEMP/unskip-closed-tests-expected-files.bin" EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.bin" ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.bin" + EXPECTED_BLOBS="$RUNNER_TEMP/unskip-closed-tests-expected-blobs.txt" + ACTUAL_BLOBS="$RUNNER_TEMP/unskip-closed-tests-actual-blobs.txt" jq -r '.pr_body' "$RESULT_PATH" > "$BODY_FILE" jq -j '.changed_files[] | .path, "\u0000", .content_sha256, "\u0000"' \ "$RESULT_PATH" > "$EXPECTED_FILES" jq -j '.changed_files[].path, "\u0000"' "$RESULT_PATH" | sort -z > "$EXPECTED_PATHS" + jq -r '[.changed_files[].content_sha256] | unique | sort | .[]' \ + "$RESULT_PATH" > "$EXPECTED_BLOBS" + find "$PACKAGE_DIRECTORY/files" -mindepth 1 -maxdepth 1 -type f \ + -printf '%f\n' | sort > "$ACTUAL_BLOBS" + cmp "$EXPECTED_BLOBS" "$ACTUAL_BLOBS" + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -not -type f -print -quit)" git diff --cached --quiet - git diff --name-only --no-renames -z | sort -z > "$ACTUAL_PATHS" - cmp "$EXPECTED_PATHS" "$ACTUAL_PATHS" while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do test -n "$path" test "${path#/}" = "$path" @@ -238,10 +398,26 @@ safe-outputs: *.cs) ;; *) echo "::error::Unexpected changed path: $path"; exit 20 ;; esac - actual_sha=$(sha256sum -- "$path") - actual_sha=${actual_sha%% *} - test "$actual_sha" = "$expected_sha" || - { echo "::error::Verified content changed for $path"; exit 20; } + git ls-files --error-unmatch -- "$path" >/dev/null + test -f "$path" + test ! -L "$path" + SOURCE_SHA=$(jq -er \ + --arg path "$path" \ + '[.candidates[] | select(.path == $path) | .source_sha256] | + unique | select(length == 1) | .[0]' \ + "$ORIGINAL_MANIFEST") + ACTUAL_SOURCE_SHA=$(sha256sum -- "$path") + ACTUAL_SOURCE_SHA=${ACTUAL_SOURCE_SHA%% *} + test "$ACTUAL_SOURCE_SHA" = "$SOURCE_SHA" || + { echo "::error::Source content changed for $path"; exit 20; } + BLOB="$PACKAGE_DIRECTORY/files/$expected_sha" + test -f "$BLOB" + test ! -L "$BLOB" + ACTUAL_BLOB_SHA=$(sha256sum -- "$BLOB") + ACTUAL_BLOB_SHA=${ACTUAL_BLOB_SHA%% *} + test "$ACTUAL_BLOB_SHA" = "$expected_sha" || + { echo "::error::Verified package content changed for $path"; exit 20; } + cp -- "$BLOB" "$path" git add -- "$path" done < "$EXPECTED_FILES" diff --git a/.github/workflows/unskip-closed-tests.lock.yml b/.github/workflows/unskip-closed-tests.lock.yml index 45b39a745b..0e4967828f 100644 --- a/.github/workflows/unskip-closed-tests.lock.yml +++ b/.github/workflows/unskip-closed-tests.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"50e4b77232c53957cb4422b202a2b71551c6003531fddcfc044383157782e054","body_hash":"8f1de6229aed153de2ce0e740ba73fdeee568b085232b4c017e5d305c134f3de","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"00e472b5dd004b0c8979cd063264259625f227f3c49e3af62477c26baae570e7","body_hash":"1de573c9bf1109cfd88d9d55ea0b60931795655ec398306d4dc13841a2ccd09b","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-dotnet","sha":"a98b56852c35b8e3190ac28c8c2271da59106c68","version":"v6.0.0 (source v6)"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"}],"mcp_servers":[{"name":"safeoutputs","tools":["apply_verified_unskips","missing_data","missing_tool","noop"]}]} # This file was automatically generated by gh-aw (v0.89.21). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -540,7 +540,7 @@ jobs: env: GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" - GH_AW_SAFE_OUTPUTS_CONFIG: "{\"apply-verified-unskips\":{\"description\":\"Revalidate selected source sites and tracking items, apply only trusted Ignore removals, require exact structured test execution evidence, and open at most one draft pull request.\",\"inputs\":{\"candidate_ids_json\":{\"default\":null,\"description\":\"JSON array of exact candidate IDs copied from the manifest.\",\"required\":true,\"type\":\"string\"},\"manifest_digest\":{\"default\":null,\"description\":\"Exact trusted manifest digest.\",\"required\":true,\"type\":\"string\"}}},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"apply-verified-unskips\":{\"description\":\"Publish the exact read-only verified Ignore-removal package in a fresh authenticated checkout and open at most one draft pull request.\",\"inputs\":{\"candidate_ids_json\":{\"default\":null,\"description\":\"JSON array of exact candidate IDs copied from the manifest.\",\"required\":true,\"type\":\"string\"},\"manifest_digest\":{\"default\":null,\"description\":\"Exact trusted manifest digest.\",\"required\":true,\"type\":\"string\"}}},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" with: script: | const path = require('path'); @@ -557,7 +557,7 @@ jobs: "repo_params": {}, "dynamic_tools": [ { - "description": "Revalidate selected source sites and tracking items, apply only trusted Ignore removals, require exact structured test execution evidence, and open at most one draft pull request.", + "description": "Publish the exact read-only verified Ignore-removal package in a fresh authenticated checkout and open at most one draft pull request.", "inputSchema": { "additionalProperties": false, "properties": { @@ -1096,6 +1096,7 @@ jobs: needs: - agent - detection + - safe_outputs if: > (!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'apply_verified_unskips') && (needs.agent.result == 'success' && needs.detection.result == 'success' && needs.detection.outputs.detection_success == 'true' && @@ -1113,86 +1114,82 @@ jobs: pattern: "{agent,agent-output-fallback}" merge-multiple: true path: ${{ runner.temp }}/gh-aw/safe-jobs/ - - name: Checkout exact analyzed revision - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + - name: Download original trusted manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json with: - fetch-depth: 0.0 - persist-credentials: true - ref: ${{ github.sha }} - - name: Set up .NET SDK - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 (source v6) + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + - name: Download verified unskip package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json with: - dotnet-version: 8.0.x - - name: Restore trusted apply tool - run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode - env: - GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json - working-directory: .github/workflows/unskip-closed-tests-tool - - name: Download original trusted manifest - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + name: unskip-closed-tests-verification-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-verification + - name: Checkout exact analyzed revision with publisher credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json with: - name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/unskip-closed-tests-manifest - - name: Revalidate, edit, and verify selected candidates - id: apply + fetch-depth: 0.0 + persist-credentials: true + ref: ${{ github.sha }} + - name: Publish one verified draft pull request run: | set -euo pipefail - set +e - dotnet run --no-restore \ - --project UnskipClosedTests.Tool.csproj \ - -- apply \ - --repo-root "$GITHUB_WORKSPACE" \ - --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ - --manifest "$ORIGINAL_MANIFEST" \ - --agent-output "$GH_AW_AGENT_OUTPUT" \ - --output "$RESULT_PATH" - APPLY_EXIT=$? - set -e - - if [ "$APPLY_EXIT" -eq 10 ]; then - rm -rf bin obj - git diff --quiet - echo "no-action=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - if [ "$APPLY_EXIT" -ne 0 ]; then - rm -rf bin obj - exit "$APPLY_EXIT" - fi test -f "$RESULT_PATH" + test -d "$PACKAGE_DIRECTORY/files" + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$ORIGINAL_MANIFEST") + test "$MANIFEST_DIGEST" != "null" jq -e \ + --arg source "$EXPECTED_COMMIT" \ + --arg digest "$MANIFEST_DIGEST" \ + --slurpfile manifest "$ORIGINAL_MANIFEST" \ '.schema_version == "1" and - .has_changes == true and - (.changed_files | length > 0) and - ([.changed_files[].path] | length) == ([.changed_files[].path] | unique | length) and - ([.changed_files[].path] | sort) == (.changed_paths | sort) and - all(.changed_files[]; - (.path | type == "string") and - (.content_sha256 | type == "string" and test("^[0-9a-f]{64}$")))' \ + .source_commit == $source and + .manifest_digest == $digest and + (.has_changes | type == "boolean") and + if .has_changes then + (.retained_candidates | length) > 0 and + ([.retained_candidates[].candidate_id] | length) == + ([.retained_candidates[].candidate_id] | unique | length) and + all(.retained_candidates[]; + . as $retained | + any($manifest[0].candidates[]; + .candidate_id == $retained.candidate_id and + .path == $retained.path and + .decision.eligible == true and + ((.owner.test_fqns | sort) == ($retained.test_fqns | sort)))) and + (.changed_files | length) > 0 and + ([.changed_files[].path] | length) == + ([.changed_files[].path] | unique | length) and + ([.changed_files[].path] | sort) == (.changed_paths | sort) and + ([.retained_candidates[].path] | unique | sort) == (.changed_paths | sort) and + all(.changed_files[]; + (.path | type == "string") and + (.content_sha256 | type == "string" and test("^[0-9a-f]{64}$"))) and + (.pr_title | type == "string" and + startswith("[unskip-closed-tests] ") and length <= 256) and + (.pr_body | type == "string" and + startswith("")) + else + (.retained_candidates | length) == 0 and + (.changed_files | length) == 0 and + (.changed_paths | length) == 0 and + .pr_title == "" and + .pr_body == "" + end' \ "$RESULT_PATH" >/dev/null - echo "no-action=false" >> "$GITHUB_OUTPUT" - echo "result-path=$RESULT_PATH" >> "$GITHUB_OUTPUT" - env: - EXPECTED_COMMIT: ${{ github.sha }} - EXPECTED_REPOSITORY: ${{ github.repository }} - GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json - GH_TOKEN: ${{ github.token }} - ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json - RESULT_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-results - RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-result.json - shell: bash - working-directory: .github/workflows/unskip-closed-tests-tool - - name: Publish one verified draft pull request - if: steps.apply.outputs.no-action == 'false' - run: | - set -euo pipefail + + if [ "$(jq -r '.has_changes' "$RESULT_PATH")" = "false" ]; then + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -print -quit)" + git diff --quiet + echo "::notice::No selected candidate passed verification." + exit 0 + fi DEFAULT_BRANCH=$(gh api "repos/${EXPECTED_REPOSITORY}" --jq '.default_branch') CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') @@ -1213,14 +1210,20 @@ jobs: EXPECTED_FILES="$RUNNER_TEMP/unskip-closed-tests-expected-files.bin" EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.bin" ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.bin" + EXPECTED_BLOBS="$RUNNER_TEMP/unskip-closed-tests-expected-blobs.txt" + ACTUAL_BLOBS="$RUNNER_TEMP/unskip-closed-tests-actual-blobs.txt" jq -r '.pr_body' "$RESULT_PATH" > "$BODY_FILE" jq -j '.changed_files[] | .path, "\u0000", .content_sha256, "\u0000"' \ "$RESULT_PATH" > "$EXPECTED_FILES" jq -j '.changed_files[].path, "\u0000"' "$RESULT_PATH" | sort -z > "$EXPECTED_PATHS" + jq -r '[.changed_files[].content_sha256] | unique | sort | .[]' \ + "$RESULT_PATH" > "$EXPECTED_BLOBS" + find "$PACKAGE_DIRECTORY/files" -mindepth 1 -maxdepth 1 -type f \ + -printf '%f\n' | sort > "$ACTUAL_BLOBS" + cmp "$EXPECTED_BLOBS" "$ACTUAL_BLOBS" + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -not -type f -print -quit)" git diff --cached --quiet - git diff --name-only --no-renames -z | sort -z > "$ACTUAL_PATHS" - cmp "$EXPECTED_PATHS" "$ACTUAL_PATHS" while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do test -n "$path" test "${path#/}" = "$path" @@ -1231,10 +1234,26 @@ jobs: *.cs) ;; *) echo "::error::Unexpected changed path: $path"; exit 20 ;; esac - actual_sha=$(sha256sum -- "$path") - actual_sha=${actual_sha%% *} - test "$actual_sha" = "$expected_sha" || - { echo "::error::Verified content changed for $path"; exit 20; } + git ls-files --error-unmatch -- "$path" >/dev/null + test -f "$path" + test ! -L "$path" + SOURCE_SHA=$(jq -er \ + --arg path "$path" \ + '[.candidates[] | select(.path == $path) | .source_sha256] | + unique | select(length == 1) | .[0]' \ + "$ORIGINAL_MANIFEST") + ACTUAL_SOURCE_SHA=$(sha256sum -- "$path") + ACTUAL_SOURCE_SHA=${ACTUAL_SOURCE_SHA%% *} + test "$ACTUAL_SOURCE_SHA" = "$SOURCE_SHA" || + { echo "::error::Source content changed for $path"; exit 20; } + BLOB="$PACKAGE_DIRECTORY/files/$expected_sha" + test -f "$BLOB" + test ! -L "$BLOB" + ACTUAL_BLOB_SHA=$(sha256sum -- "$BLOB") + ACTUAL_BLOB_SHA=${ACTUAL_BLOB_SHA%% *} + test "$ACTUAL_BLOB_SHA" = "$expected_sha" || + { echo "::error::Verified package content changed for $path"; exit 20; } + cp -- "$BLOB" "$path" git add -- "$path" done < "$EXPECTED_FILES" @@ -1303,7 +1322,9 @@ jobs: EXPECTED_REPOSITORY: ${{ github.repository }} GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json GH_TOKEN: ${{ github.token }} - RESULT_PATH: ${{ steps.apply.outputs.result-path }} + ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json + PACKAGE_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-verification + RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-verification/result.json shell: bash collect-unskip-candidates: @@ -1407,6 +1428,7 @@ jobs: - collect-unskip-candidates - detection - safe_outputs + - verify-selected-unskips if: > always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || needs.activation.outputs.oauth_token_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true' || @@ -2042,11 +2064,12 @@ jobs: - activation - agent - detection + - verify-selected-unskips if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' runs-on: ubuntu-slim permissions: issues: write - timeout-minutes: 45 + timeout-minutes: 15 env: GH_AW_AGENT_AIC: ${{ needs.agent.outputs.aic }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} @@ -2152,3 +2175,160 @@ jobs: /tmp/gh-aw/temporary-id-map.json /tmp/gh-aw/safe-output-errors.json if-no-files-found: ignore + + verify-selected-unskips: + name: Verify selected unskips without write credentials + needs: + - agent + - detection + if: > + needs.agent.result == 'success' && needs.detection.result == 'success' && needs.detection.outputs.detection_success == 'true' && + contains(needs.agent.outputs.output_types, 'apply_verified_unskips') + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + timeout-minutes: 45 + steps: + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash + run: | # zizmor: ignore[github-env] - GITHUB_SERVER_URL is set by GitHub Actions, not user input. + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Download agent output artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + merge-multiple: true + path: ${{ runner.temp }}/gh-aw/verify-job + pattern: "{agent,agent-output-fallback}" + - name: Checkout exact analyzed revision without credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + with: + fetch-depth: 0.0 + persist-credentials: false + ref: ${{ github.sha }} + - name: Set up .NET SDK + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 (source v6) + with: + dotnet-version: 8.0.x + - name: Restore trusted apply tool + run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode + working-directory: .github/workflows/unskip-closed-tests-tool + - name: Download original trusted manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + - name: Revalidate, edit, verify, and package selected candidates + run: | + set -euo pipefail + + rm -rf "$PACKAGE_DIRECTORY" + mkdir -p "$PACKAGE_DIRECTORY/files" + + set +e + timeout --kill-after=30s 40m dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- apply \ + --repo-root "$GITHUB_WORKSPACE" \ + --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ + --manifest "$ORIGINAL_MANIFEST" \ + --agent-output "$AGENT_OUTPUT" \ + --output "$RESULT_PATH" + APPLY_EXIT=$? + set -e + + if [ "$APPLY_EXIT" -ne 0 ] && [ "$APPLY_EXIT" -ne 10 ]; then + rm -rf bin obj + exit "$APPLY_EXIT" + fi + + test -f "$RESULT_PATH" + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$ORIGINAL_MANIFEST") + test "$MANIFEST_DIGEST" != "null" + jq -e \ + --arg source "$EXPECTED_COMMIT" \ + --arg digest "$MANIFEST_DIGEST" \ + '.schema_version == "1" and + .source_commit == $source and + .manifest_digest == $digest' \ + "$RESULT_PATH" >/dev/null + + rm -rf bin obj + git diff --cached --quiet + + if [ "$APPLY_EXIT" -eq 10 ]; then + jq -e \ + '.has_changes == false and + (.retained_candidates | length) == 0 and + (.changed_files | length) == 0 and + (.changed_paths | length) == 0' \ + "$RESULT_PATH" >/dev/null + git diff --quiet + else + jq -e \ + '.has_changes == true and + (.retained_candidates | length) > 0 and + (.changed_files | length) > 0 and + ([.changed_files[].path] | length) == ([.changed_files[].path] | unique | length) and + ([.changed_files[].path] | sort) == (.changed_paths | sort) and + all(.changed_files[]; + (.path | type == "string") and + (.content_sha256 | type == "string" and test("^[0-9a-f]{64}$")))' \ + "$RESULT_PATH" >/dev/null + + EXPECTED_FILES="$RUNNER_TEMP/unskip-closed-tests-expected-files.bin" + EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.bin" + ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.bin" + jq -j '.changed_files[] | .path, "\u0000", .content_sha256, "\u0000"' \ + "$RESULT_PATH" > "$EXPECTED_FILES" + jq -j '.changed_files[].path, "\u0000"' "$RESULT_PATH" | sort -z > "$EXPECTED_PATHS" + git diff --name-only --no-renames -z | sort -z > "$ACTUAL_PATHS" + cmp "$EXPECTED_PATHS" "$ACTUAL_PATHS" + + while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do + test -n "$path" + test "${path#/}" = "$path" + case "/$path/" in + *"/../"*|*"/./"*) exit 20 ;; + esac + case "$path" in + *.cs) ;; + *) echo "::error::Unexpected changed path: $path"; exit 20 ;; + esac + actual_sha=$(sha256sum -- "$GITHUB_WORKSPACE/$path") + actual_sha=${actual_sha%% *} + test "$actual_sha" = "$expected_sha" || + { echo "::error::Verified content changed for $path"; exit 20; } + blob="$PACKAGE_DIRECTORY/files/$expected_sha" + if [ -e "$blob" ]; then + cmp "$GITHUB_WORKSPACE/$path" "$blob" + else + cp -- "$GITHUB_WORKSPACE/$path" "$blob" + fi + done < "$EXPECTED_FILES" + fi + + cp "$RESULT_PATH" "$PACKAGE_DIRECTORY/result.json" + env: + AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/verify-job/agent_output.json + EXPECTED_COMMIT: ${{ github.sha }} + EXPECTED_REPOSITORY: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json + PACKAGE_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-verification + RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-result.json + shell: bash + working-directory: .github/workflows/unskip-closed-tests-tool + - name: Upload verified unskip package + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) + with: + if-no-files-found: error + name: unskip-closed-tests-verification-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-verification + retention-days: 1.0 diff --git a/.github/workflows/unskip-closed-tests.md b/.github/workflows/unskip-closed-tests.md index 770225d7b8..b344d1c336 100644 --- a/.github/workflows/unskip-closed-tests.md +++ b/.github/workflows/unskip-closed-tests.md @@ -32,6 +32,9 @@ jobs: needs: [collect-unskip-candidates] if: needs.collect-unskip-candidates.outputs.eligible-count != '0' +safe-outputs: + timeout-minutes: 15 + imports: - unskip-closed-tests-prepare.md - unskip-closed-tests-shared.md @@ -76,6 +79,7 @@ When one or more manifest candidates are safe to attempt, call - `candidate_ids_json` containing a JSON array of unique candidate IDs copied exactly from the manifest. -The trusted safe-output job revalidates the source, remote state, proposed -sites, edits, and structured test evidence before publishing one draft pull +The trusted read-only verification job revalidates the source, remote state, +proposed sites, edits, and structured test evidence, then a fresh publisher +checkout applies only the validated package before opening one draft pull request. If no candidate should proceed, call `noop` once with a short reason.