diff --git a/.github/agents/unskip-closed-tests.agent.md b/.github/agents/unskip-closed-tests.agent.md new file mode 100644 index 0000000000..cb98481cf5 --- /dev/null +++ b/.github/agents/unskip-closed-tests.agent.md @@ -0,0 +1,50 @@ +--- +name: unskip-closed-tests +description: "Selects only source-bound, deterministically eligible .NET Ignore sites for trusted revalidation and test execution." +--- + +# Unskip Closed Tests Planner + +You are a read-only planner. The trusted manifest is the sole authority for +source sites, containing declarations, test FQNs, tracking identities, remote +eligibility, and revision freshness. + +## Required process + +1. Read `GH_AW_UNSKIP_MANIFEST` with `jq`. +2. Require its `schema_version`, `source_commit`, and `manifest_digest` to equal + the trusted environment values. +3. Consider only candidates where `decision.eligible` is `true`. +4. Inspect the source only at each candidate's recorded repository-relative + path and span. Use it to identify ambiguity, never to create a replacement + identity. +5. Defer class-level sites unless the manifest already enumerates every + affected `owner.test_fqns` entry and has no class-level deferral. +6. Select only IDs copied byte-for-byte from `candidate_id`. +7. Call exactly one allowed output and stop. + +## Mandatory deferrals + +Defer any candidate when: + +- its source path, span, owner, containing type chain, declaration identity, + test FQN, issue identity, or state appears inconsistent; +- a method's recorded owner is not its actual syntax ancestor; +- class-level inheritance, nesting, partial declarations, or incomplete test + enumeration is present; +- issue context does not clearly correspond to the ignored test even though + the deterministic remote state is eligible; +- the candidate depends on an inferred anchor, source rewrite, or remote fact. + +Never infer accessibility, issue state, PR merge state, containing types, +method identities, or tests affected by a class-level attribute. + +## Output + +For one or more selected candidates, call `apply_verified_unskips` once with +the exact manifest digest and a JSON array string of unique candidate IDs. The +safe-output job may retain fewer candidates after source, remote, build, and +TRX revalidation. + +When no candidate remains, call `noop` once. Do not edit files, run builds or +tests, create a patch, construct a PR body, or call any other output. diff --git a/.github/scripts/test_unskip_closed_tests_verify.ps1 b/.github/scripts/test_unskip_closed_tests_verify.ps1 new file mode 100644 index 0000000000..0f17da8d5d --- /dev/null +++ b/.github/scripts/test_unskip_closed_tests_verify.ps1 @@ -0,0 +1,348 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$hookPath = Join-Path $PSScriptRoot '../workflows/unskip-closed-tests-verify.ps1' +. $hookPath -RequestPath 'unused' + +$script:Passed = 0 + +function Assert-Equal { + param( + [Parameter(Mandatory)] $Expected, + [Parameter(Mandatory)] $Actual, + [Parameter(Mandatory)] [string] $Message + ) + + if ($Expected -ne $Actual) { + throw "$Message Expected '$Expected', actual '$Actual'." + } +} + +function Assert-Contains { + param( + [Parameter(Mandatory)] [object[]] $Values, + [Parameter(Mandatory)] [object] $Expected, + [Parameter(Mandatory)] [string] $Message + ) + + if ($Values -notcontains $Expected) { + throw "$Message Missing '$Expected'." + } +} + +function Assert-Throws { + param( + [Parameter(Mandatory)] [scriptblock] $Action, + [Parameter(Mandatory)] [string] $Pattern + ) + + try { + & $Action + } + catch { + if ($_.Exception.Message -notmatch $Pattern) { + throw "Exception '$($_.Exception.Message)' did not match '$Pattern'." + } + return + } + + throw "Expected exception matching '$Pattern'." +} + +function Invoke-TestCase { + param( + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [scriptblock] $Test + ) + + & $Test + $script:Passed++ + Write-Host "PASS: $Name" +} + +function Write-Request { + param( + [Parameter(Mandatory)] [string] $Path, + [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]] $Tests, + [string] $SourceCommit = ('a' * 40) + ) + + @{ + schema_version = '1' + repository = 'microsoft/testfx' + source_commit = $SourceCommit + candidate = @{ candidate_id = 'candidate-1' } + tests = $Tests + } | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $Path -Encoding utf8NoBOM +} + +function Save-Functions { + param([Parameter(Mandatory)] [string[]] $Names) + + $saved = @{} + foreach ($name in $Names) { + $saved[$name] = (Get-Item "Function:$name").ScriptBlock + } + return $saved +} + +function Restore-Functions { + param([Parameter(Mandatory)] [hashtable] $Saved) + + foreach ($entry in $Saved.GetEnumerator()) { + Set-Item "Function:$($entry.Key)" -Value $entry.Value + } +} + +$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) "testfx-unskip-hook-$PID" +Remove-Item -LiteralPath $temporaryRoot -Recurse -Force -ErrorAction SilentlyContinue +[void] (New-Item -ItemType Directory -Path $temporaryRoot) + +try { + Invoke-TestCase 'parses exact source and test identities' { + $requestPath = Join-Path $temporaryRoot 'valid-request.json' + Write-Request -Path $requestPath -Tests @( + @{ + fqn = 'Example.Tests.TestOne' + source_path = 'test/UnitTests/Example/Tests.cs' + result_file = (Join-Path $temporaryRoot 'TestOne.trx') + } + ) + $request = Read-VerificationRequest -Path $requestPath + Assert-Equal -Expected 'candidate-1' -Actual $request.CandidateId -Message 'Candidate parsing failed.' + Assert-Equal -Expected ('a' * 40) -Actual $request.SourceCommit -Message 'Commit parsing failed.' + Assert-Equal -Expected 'Example.Tests.TestOne' -Actual $request.Tests[0].Fqn -Message 'FQN parsing failed.' + } + + Invoke-TestCase 'rejects malformed and duplicate test identities' { + $emptyPath = Join-Path $temporaryRoot 'empty-request.json' + Write-Request -Path $emptyPath -Tests @() + Assert-Throws -Action { Read-VerificationRequest -Path $emptyPath } -Pattern 'must not be empty' + + $duplicatePath = Join-Path $temporaryRoot 'duplicate-request.json' + $test = @{ + fqn = 'Example.Tests.TestOne' + source_path = 'test/UnitTests/Example/Tests.cs' + result_file = (Join-Path $temporaryRoot 'TestOne.trx') + } + Write-Request -Path $duplicatePath -Tests @($test, $test) + Assert-Throws -Action { Read-VerificationRequest -Path $duplicatePath } -Pattern 'duplicate test identity' + + $fabricatedPath = Join-Path $temporaryRoot 'fabricated-request.json' + Write-Request -Path $fabricatedPath -Tests @( + @{ + fqn = 'Invented' + source_path = 'test/UnitTests/Example/Tests.cs' + result_file = (Join-Path $temporaryRoot 'Invented.trx') + } + ) + Assert-Throws -Action { Read-VerificationRequest -Path $fabricatedPath } -Pattern 'supported test identity' + } + + Invoke-TestCase 'maps source to the nearest unambiguous project' { + $source = Join-Path $temporaryRoot 'project-map/test/UnitTests/Example/Tests.cs' + [void] (New-Item -ItemType Directory -Path (Split-Path -Parent $source) -Force) + Set-Content -LiteralPath $source -Value 'class Tests {}' -Encoding utf8NoBOM + $project = Join-Path (Split-Path -Parent $source) 'Example.csproj' + Set-Content -LiteralPath $project -Value '' -Encoding utf8NoBOM + Assert-Equal -Expected $project -Actual ( + Get-TestProject -Root (Join-Path $temporaryRoot 'project-map') -SourcePath 'test/UnitTests/Example/Tests.cs' + ) -Message 'Nearest project mapping failed.' + } + + Invoke-TestCase 'rejects ambiguous and unmapped source projects' { + $ambiguousRoot = Join-Path $temporaryRoot 'ambiguous' + $source = Join-Path $ambiguousRoot 'test/Example/Tests.cs' + [void] (New-Item -ItemType Directory -Path (Split-Path -Parent $source) -Force) + Set-Content -LiteralPath $source -Value 'class Tests {}' -Encoding utf8NoBOM + Set-Content -LiteralPath (Join-Path (Split-Path -Parent $source) 'One.csproj') -Value '' -Encoding utf8NoBOM + Set-Content -LiteralPath (Join-Path (Split-Path -Parent $source) 'Two.csproj') -Value '' -Encoding utf8NoBOM + Assert-Throws -Action { + Get-TestProject -Root $ambiguousRoot -SourcePath 'test/Example/Tests.cs' + } -Pattern 'ambiguous' + + $unmappedRoot = Join-Path $temporaryRoot 'unmapped' + $unmappedSource = Join-Path $unmappedRoot 'test/Example/Tests.cs' + [void] (New-Item -ItemType Directory -Path (Split-Path -Parent $unmappedSource) -Force) + Set-Content -LiteralPath $unmappedSource -Value 'class Tests {}' -Encoding utf8NoBOM + Assert-Throws -Action { + Get-TestProject -Root $unmappedRoot -SourcePath 'test/Example/Tests.cs' + } -Pattern 'No owning' + } + + Invoke-TestCase 'selects every portable target framework and rejects mixed unsupported targets' { + Assert-Equal -Expected 'net8.0,net10.0' -Actual ( + (Select-TargetFrameworks -Frameworks @('net10.0', 'net8.0', 'net8.0')) -join ',' + ) -Message 'Portable framework ordering failed.' + Assert-Throws -Action { + Select-TargetFrameworks -Frameworks @('net462', 'net8.0') + } -Pattern 'Cannot verify every target framework' + Assert-Throws -Action { + Select-TargetFrameworks -Frameworks @('net8.0-windows') + } -Pattern 'Cannot verify every target framework' + } + + Invoke-TestCase 'uses distinct TRX files for multi-target verification' { + $requested = Join-Path $temporaryRoot 'results/TestOne.trx' + Assert-Equal -Expected $requested -Actual ( + Get-TargetResultFile -RequestedResultFile $requested -TargetFramework 'net8.0' -TargetFrameworkCount 1 + ) -Message 'Single-target result path changed.' + Assert-Equal -Expected (Join-Path $temporaryRoot 'results/TestOne--net9.0.trx') -Actual ( + Get-TargetResultFile -RequestedResultFile $requested -TargetFramework 'net9.0' -TargetFrameworkCount 2 + ) -Message 'Multi-target result path was not framework-specific.' + } + + Invoke-TestCase 'builds exact-FQN and requested-TRX commands' { + $project = Join-Path $temporaryRoot 'Example.csproj' + $result = Join-Path $temporaryRoot 'results/TestOne.trx' + $buildArguments = Get-BuildArguments -Project $project -TargetFramework 'net8.0' + Assert-Contains -Values $buildArguments -Expected '-p:EnableCodeCoverage=False' -Message 'Build coverage opt-out failed.' + Assert-Contains -Values $buildArguments -Expected '-bl:{}' -Message 'Build binlog argument missing.' + + $testArguments = Get-TestArguments -Project $project -TargetFramework 'net8.0' -Fqn 'Example.Tests.TestOne' -ResultFile $result + Assert-Contains -Values $testArguments -Expected '--filter-uid' -Message 'Test UID filter missing.' + Assert-Contains -Values $testArguments -Expected 'Example.Tests.TestOne' -Message 'Exact FQN missing.' + Assert-Contains -Values $testArguments -Expected ([System.IO.Path]::GetFileName($result)) -Message 'Requested TRX filename missing.' + Assert-Contains -Values $testArguments -Expected ([System.IO.Path]::GetDirectoryName($result)) -Message 'Requested TRX directory missing.' + Assert-Contains -Values $testArguments -Expected '-bl:{}' -Message 'Test binlog argument missing.' + } + + Invoke-TestCase 'rejects stale source revisions before execution' { + $repository = Join-Path $temporaryRoot 'stale-repository' + [void] (New-Item -ItemType Directory -Path $repository) + & git -C $repository init --quiet + & git -C $repository config user.email tests@example.invalid + & git -C $repository config user.name Tests + Set-Content -LiteralPath (Join-Path $repository 'README.md') -Value 'fixture' -Encoding utf8NoBOM + & git -C $repository add . + & git -C $repository commit --quiet -m fixture + Assert-Throws -Action { + Assert-Revision -Root $repository -ExpectedCommit ('0' * 40) + } -Pattern 'Repository revision changed' + } + + Invoke-TestCase 'allows only runner temp and dedicated Git metadata results' { + $repository = Join-Path $temporaryRoot 'result-roots' + $runnerTemp = Join-Path $temporaryRoot 'runner-temp' + [void] (New-Item -ItemType Directory -Path $repository) + [void] (New-Item -ItemType Directory -Path $runnerTemp) + & git -C $repository init --quiet + + $previousRunnerTemp = $env:RUNNER_TEMP + $env:RUNNER_TEMP = $runnerTemp + try { + $runnerResult = Join-Path $runnerTemp 'runner.trx' + Assert-Equal -Expected ([System.IO.Path]::GetFullPath($runnerResult)) -Actual ( + Resolve-ResultPath -Root $repository -Value $runnerResult + ) -Message 'Runner temp result path was rejected.' + + $gitDirectory = & git -C $repository rev-parse --git-dir + $metadataResult = Join-Path $repository "$gitDirectory/unskip-closed-tests/digest/candidate/result.trx" + Assert-Equal -Expected ([System.IO.Path]::GetFullPath($metadataResult)) -Actual ( + Resolve-ResultPath -Root $repository -Value $metadataResult + ) -Message 'Dedicated Git metadata result path was rejected.' + + $outsideResult = Join-Path $temporaryRoot 'outside/result.trx' + Assert-Throws -Action { + Resolve-ResultPath -Root $repository -Value $outsideResult + } -Pattern 'outside the trusted output roots' + } + finally { + $env:RUNNER_TEMP = $previousRunnerTemp + } + } + + Invoke-TestCase 'fails closed when the requested TRX is missing' { + $root = Join-Path $temporaryRoot 'missing-trx' + $source = Join-Path $root 'test/Example/Tests.cs' + [void] (New-Item -ItemType Directory -Path (Split-Path -Parent $source) -Force) + Set-Content -LiteralPath $source -Value 'class Tests {}' -Encoding utf8NoBOM + $project = Join-Path (Split-Path -Parent $source) 'Example.csproj' + Set-Content -LiteralPath $project -Value '' -Encoding utf8NoBOM + $requestPath = Join-Path $root 'request.json' + $resultPath = Join-Path $root 'results/missing.trx' + Write-Request -Path $requestPath -Tests @( + @{ + fqn = 'Example.Tests.TestOne' + source_path = 'test/Example/Tests.cs' + result_file = $resultPath + } + ) + + $saved = Save-Functions -Names @( + 'Assert-Revision', + 'Initialize-RepositoryBuild', + 'Get-ProjectTargetFrameworks', + 'Get-DotNetPath', + 'Invoke-CheckedProcess' + ) + try { + Set-Item Function:Assert-Revision -Value { param($Root, $ExpectedCommit) } + Set-Item Function:Initialize-RepositoryBuild -Value { param($Root, $SourceCommit, $RequiresPack, $Timeout) } + Set-Item Function:Get-ProjectTargetFrameworks -Value { param($Root, $Project, $Timeout) @('net8.0') } + Set-Item Function:Get-DotNetPath -Value { param($Root) 'dotnet' } + Set-Item Function:Invoke-CheckedProcess -Value { param($FileName, $Arguments, $WorkingDirectory, $Timeout) } + $previousRunnerTemp = $env:RUNNER_TEMP + $env:RUNNER_TEMP = $root + try { + Assert-Throws -Action { + Invoke-UnskipVerification -Path $requestPath -Root $root -Timeout 30 + } -Pattern 'did not create requested TRX' + } + finally { + $env:RUNNER_TEMP = $previousRunnerTemp + } + } + finally { + Restore-Functions -Saved $saved + } + } + + Invoke-TestCase 'runs repository pack only once for acceptance projects' { + $root = Join-Path $temporaryRoot 'acceptance' + [void] (New-Item -ItemType Directory -Path $root) + $buildScript = Join-Path $root $(if ($IsWindows) { 'build.cmd' } else { 'build.sh' }) + Set-Content -LiteralPath $buildScript -Value '' -Encoding utf8NoBOM + $runnerTemp = Join-Path $root 'runner-temp' + $calls = [System.Collections.Generic.List[object]]::new() + $saved = Save-Functions -Names @('Invoke-CheckedProcess', 'Assert-Revision') + try { + Set-Item Function:Invoke-CheckedProcess -Value { + param($FileName, $Arguments, $WorkingDirectory, $Timeout) + $calls.Add([pscustomobject]@{ FileName = $FileName; Arguments = @($Arguments) }) + } + Set-Item Function:Assert-Revision -Value { param($Root, $ExpectedCommit) } + $previousRunnerTemp = $env:RUNNER_TEMP + $env:RUNNER_TEMP = $runnerTemp + try { + Initialize-RepositoryBuild -Root $root -SourceCommit ('a' * 40) -RequiresPack $true -Timeout 30 + Initialize-RepositoryBuild -Root $root -SourceCommit ('a' * 40) -RequiresPack $true -Timeout 30 + } + finally { + $env:RUNNER_TEMP = $previousRunnerTemp + } + } + finally { + Restore-Functions -Saved $saved + } + Assert-Equal -Expected 1 -Actual $calls.Count -Message 'Repository pack should run once.' + Assert-Contains -Values $calls[0].Arguments -Expected '-pack' -Message 'Acceptance pack argument missing.' + } + + Invoke-TestCase 'replaces the packaged fail-closed placeholder' { + $hookText = Get-Content -LiteralPath $hookPath -Raw + if ($hookText.Contains('The repository must replace verification.command')) { + throw 'The TestFX hook still contains the package placeholder.' + } + $config = Get-Content -LiteralPath (Join-Path $PSScriptRoot '../workflows/unskip-closed-tests.config.json') -Raw | + ConvertFrom-Json -Depth 16 + Assert-Equal -Expected 'pwsh' -Actual $config.verification.command[0] -Message 'PowerShell command is not configured.' + Assert-Contains -Values @($config.verification.command) -Expected '.github/workflows/unskip-closed-tests-verify.ps1' -Message 'PowerShell hook path is not configured.' + } + + Assert-Equal -Expected 12 -Actual $script:Passed -Message 'Unexpected hook test count.' + Write-Host "All $script:Passed unskip closed tests PowerShell hook tests passed." +} +finally { + Remove-Item -LiteralPath $temporaryRoot -Recurse -Force -ErrorAction SilentlyContinue +} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index dee60bc072..70e879b560 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -349,7 +349,7 @@ the cause. | [`resource-lock-refactoring.md`](./resource-lock-refactoring.md) | Daily + manual | Prepares one bounded test project for safe parallel execution by eliminating shared state or applying the narrowest appropriate `[ResourceLock]`, then opens a draft PR. | | [`repository-quality-improver.md`](./repository-quality-improver.md) | Weekday schedule + manual | Daily analysis of repository quality, rotating focus areas. Opens tracking issues like this one. | | [`daily-file-diet.md`](./daily-file-diet.md) | Daily + manual | Identifies oversized source files and opens actionable refactoring issues. | -| [`unskip-closed-tests.md`](./unskip-closed-tests.md) | Weekly + manual | Finds tests skipped via `[Ignore("…#issue")]` whose tracking issue is now closed, verifies they pass, and opens a PR re-enabling them. | +| [`unskip-closed-tests.md`](./unskip-closed-tests.md) | Weekly + manual | Builds a source-bound Ignore inventory at the exact commit, accepts only completed issues or merged PRs, and opens one draft PR only for tests proven executed and passed in structured TRX results. | | [`duplicate-code-detector.md`](./duplicate-code-detector.md) | Schedule + manual | Identifies duplicate code patterns and suggests refactoring opportunities. | | [`malicious-code-scan.md`](./malicious-code-scan.md) | Schedule + manual | Reviews code changes from the last 3 days for suspicious patterns indicating malicious or agentic threats. | | [`markdown-linter.md`](./markdown-linter.md) | Schedule + manual | Runs Markdown quality checks using markdownlint-cli2 and opens issues for violations. | diff --git a/.github/workflows/test-unskip-closed-tests.yml b/.github/workflows/test-unskip-closed-tests.yml new file mode 100644 index 0000000000..d8e9bae173 --- /dev/null +++ b/.github/workflows/test-unskip-closed-tests.yml @@ -0,0 +1,50 @@ +name: Test unskip closed tests helper + +on: + pull_request: + paths: + - '.github/scripts/test_unskip_closed_tests_verify.ps1' + - '.github/agents/unskip-closed-tests.agent.md' + - '.github/workflows/unskip-closed-tests.config.json' + - '.github/workflows/unskip-closed-tests-prepare.md' + - '.github/workflows/unskip-closed-tests-shared.md' + - '.github/workflows/unskip-closed-tests-tool/**' + - '.github/workflows/unskip-closed-tests-verify.ps1' + - '.github/workflows/test-unskip-closed-tests.yml' + - '.github/workflows/unskip-closed-tests.md' + push: + branches: + - main + - 'rel/*' + paths: + - '.github/scripts/test_unskip_closed_tests_verify.ps1' + - '.github/agents/unskip-closed-tests.agent.md' + - '.github/workflows/unskip-closed-tests.config.json' + - '.github/workflows/unskip-closed-tests-prepare.md' + - '.github/workflows/unskip-closed-tests-shared.md' + - '.github/workflows/unskip-closed-tests-tool/**' + - '.github/workflows/unskip-closed-tests-verify.ps1' + - '.github/workflows/test-unskip-closed-tests.yml' + - '.github/workflows/unskip-closed-tests.md' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: test-unskip-closed-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test unskip closed tests helper + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + - run: pwsh -NoLogo -NoProfile -File .github/scripts/test_unskip_closed_tests_verify.ps1 + - run: dotnet run --project Tests/UnskipClosedTests.Tool.Tests.csproj + working-directory: .github/workflows/unskip-closed-tests-tool diff --git a/.github/workflows/unskip-closed-tests-prepare.md b/.github/workflows/unskip-closed-tests-prepare.md new file mode 100644 index 0000000000..e51d858e52 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-prepare.md @@ -0,0 +1,485 @@ +--- +description: >- + Deterministic source inventory, GitHub eligibility resolution, and trusted + safe-output publication for Unskip Closed Tests. + +jobs: + collect-unskip-candidates: + name: Collect verified unskip candidates + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: read + pull-requests: read + outputs: + eligible-count: ${{ steps.collect.outputs.eligible-count }} + source-commit: ${{ steps.collect.outputs.source-commit }} + manifest-digest: ${{ steps.collect.outputs.manifest-digest }} + steps: + - name: Checkout trusted source revision + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "8.0.x" + + - name: Restore trusted inventory tool + working-directory: .github/workflows/unskip-closed-tests-tool + run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode + + - name: Inventory source and resolve tracking items + id: collect + shell: bash + working-directory: .github/workflows/unskip-closed-tests-tool + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_COMMIT: ${{ github.sha }} + RAW_INVENTORY: ${{ runner.temp }}/unskip-closed-tests-inventory.json + RESOLVED_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest.json + run: | + set -euo pipefail + + set +e + dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- inventory \ + --repo-root "$GITHUB_WORKSPACE" \ + --repository "$EXPECTED_REPOSITORY" \ + --source-commit "$EXPECTED_COMMIT" \ + --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ + --output "$RAW_INVENTORY" + INVENTORY_EXIT=$? + set -e + if [ "$INVENTORY_EXIT" -ne 0 ] && [ "$INVENTORY_EXIT" -ne 10 ]; then + exit "$INVENTORY_EXIT" + fi + + set +e + dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- resolve \ + --manifest "$RAW_INVENTORY" \ + --output "$RESOLVED_MANIFEST" + RESOLVE_EXIT=$? + set -e + if [ "$RESOLVE_EXIT" -ne 0 ] && [ "$RESOLVE_EXIT" -ne 10 ]; then + exit "$RESOLVE_EXIT" + fi + + ELIGIBLE_COUNT=$(jq -r '[.candidates[] | select(.decision.eligible == true)] | length' "$RESOLVED_MANIFEST") + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$RESOLVED_MANIFEST") + test "$MANIFEST_DIGEST" != "null" + cp "$RESOLVED_MANIFEST" "$GITHUB_WORKSPACE/manifest.json" + { + echo "eligible-count=$ELIGIBLE_COUNT" + echo "source-commit=$EXPECTED_COMMIT" + echo "manifest-digest=$MANIFEST_DIGEST" + } >> "$GITHUB_OUTPUT" + + - name: Upload trusted candidate manifest + uses: actions/upload-artifact@v7 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: manifest.json + if-no-files-found: error + retention-days: 1 + + verify-selected-unskips: + name: Verify selected unskips without write credentials + needs: [agent, detection] + if: >- + needs.agent.result == 'success' && + needs.detection.result == 'success' && + needs.detection.outputs.detection_success == 'true' && + contains(needs.agent.outputs.output_types, 'apply_verified_unskips') + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + issues: read + pull-requests: read + steps: + - name: Download agent output artifact + uses: actions/download-artifact@v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: ${{ runner.temp }}/gh-aw/verify-job + + - name: Checkout exact analyzed revision without credentials + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "8.0.x" + + - name: Restore trusted apply tool + working-directory: .github/workflows/unskip-closed-tests-tool + run: dotnet restore UnskipClosedTests.Tool.csproj --locked-mode + + - name: Download original trusted manifest + uses: actions/download-artifact@v8.0.1 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + + - name: Revalidate, edit, verify, and package selected candidates + shell: bash + working-directory: .github/workflows/unskip-closed-tests-tool + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_COMMIT: ${{ github.sha }} + AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/verify-job/agent_output.json + ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json + RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-result.json + PACKAGE_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-verification + run: | + set -euo pipefail + + rm -rf "$PACKAGE_DIRECTORY" + mkdir -p "$PACKAGE_DIRECTORY/files" + + set +e + timeout --kill-after=30s 40m dotnet run --no-restore \ + --project UnskipClosedTests.Tool.csproj \ + -- apply \ + --repo-root "$GITHUB_WORKSPACE" \ + --config "$GITHUB_WORKSPACE/.github/workflows/unskip-closed-tests.config.json" \ + --manifest "$ORIGINAL_MANIFEST" \ + --agent-output "$AGENT_OUTPUT" \ + --output "$RESULT_PATH" + APPLY_EXIT=$? + set -e + + if [ "$APPLY_EXIT" -ne 0 ] && [ "$APPLY_EXIT" -ne 10 ]; then + rm -rf bin obj + exit "$APPLY_EXIT" + fi + + test -f "$RESULT_PATH" + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$ORIGINAL_MANIFEST") + test "$MANIFEST_DIGEST" != "null" + jq -e \ + --arg source "$EXPECTED_COMMIT" \ + --arg digest "$MANIFEST_DIGEST" \ + '.schema_version == "1" and + .source_commit == $source and + .manifest_digest == $digest' \ + "$RESULT_PATH" >/dev/null + + rm -rf bin obj + git diff --cached --quiet + + if [ "$APPLY_EXIT" -eq 10 ]; then + jq -e \ + '.has_changes == false and + (.retained_candidates | length) == 0 and + (.changed_files | length) == 0 and + (.changed_paths | length) == 0' \ + "$RESULT_PATH" >/dev/null + git diff --quiet + else + jq -e \ + '.has_changes == true and + (.retained_candidates | length) > 0 and + (.changed_files | length) > 0 and + ([.changed_files[].path] | length) == ([.changed_files[].path] | unique | length) and + ([.changed_files[].path] | sort) == (.changed_paths | sort) and + all(.changed_files[]; + (.path | type == "string") and + (.content_sha256 | type == "string" and test("^[0-9a-f]{64}$")))' \ + "$RESULT_PATH" >/dev/null + + EXPECTED_FILES="$RUNNER_TEMP/unskip-closed-tests-expected-files.bin" + EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.bin" + ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.bin" + jq -j '.changed_files[] | .path, "\u0000", .content_sha256, "\u0000"' \ + "$RESULT_PATH" > "$EXPECTED_FILES" + jq -j '.changed_files[].path, "\u0000"' "$RESULT_PATH" | sort -z > "$EXPECTED_PATHS" + git diff --name-only --no-renames -z | sort -z > "$ACTUAL_PATHS" + cmp "$EXPECTED_PATHS" "$ACTUAL_PATHS" + + while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do + test -n "$path" + test "${path#/}" = "$path" + case "/$path/" in + *"/../"*|*"/./"*) exit 20 ;; + esac + case "$path" in + *.cs) ;; + *) echo "::error::Unexpected changed path: $path"; exit 20 ;; + esac + actual_sha=$(sha256sum -- "$GITHUB_WORKSPACE/$path") + actual_sha=${actual_sha%% *} + test "$actual_sha" = "$expected_sha" || + { echo "::error::Verified content changed for $path"; exit 20; } + blob="$PACKAGE_DIRECTORY/files/$expected_sha" + if [ -e "$blob" ]; then + cmp "$GITHUB_WORKSPACE/$path" "$blob" + else + cp -- "$GITHUB_WORKSPACE/$path" "$blob" + fi + done < "$EXPECTED_FILES" + fi + + cp "$RESULT_PATH" "$PACKAGE_DIRECTORY/result.json" + + - name: Upload verified unskip package + uses: actions/upload-artifact@v7 + with: + name: unskip-closed-tests-verification-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-verification + if-no-files-found: error + retention-days: 1 + +safe-outputs: + needs: [verify-selected-unskips] + jobs: + apply-verified-unskips: + description: >- + Publish the exact read-only verified Ignore-removal package in a fresh + authenticated checkout and open at most one draft pull request. + needs: safe_outputs + if: >- + needs.agent.result == 'success' && + needs.detection.result == 'success' && + needs.detection.outputs.detection_success == 'true' && + contains(needs.agent.outputs.output_types, 'apply_verified_unskips') + runs-on: ubuntu-latest + permissions: + contents: write + issues: read + pull-requests: write + inputs: + manifest_digest: + description: "Exact trusted manifest digest." + required: true + type: string + candidate_ids_json: + description: "JSON array of exact candidate IDs copied from the manifest." + required: true + type: string + steps: + - name: Download original trusted manifest + uses: actions/download-artifact@v8.0.1 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + + - name: Download verified unskip package + uses: actions/download-artifact@v8.0.1 + with: + name: unskip-closed-tests-verification-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-verification + + - name: Checkout exact analyzed revision with publisher credentials + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: true + + - name: Publish one verified draft pull request + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_COMMIT: ${{ github.sha }} + ORIGINAL_MANIFEST: ${{ runner.temp }}/unskip-closed-tests-manifest/manifest.json + PACKAGE_DIRECTORY: ${{ runner.temp }}/unskip-closed-tests-verification + RESULT_PATH: ${{ runner.temp }}/unskip-closed-tests-verification/result.json + run: | + set -euo pipefail + + test -f "$RESULT_PATH" + test -d "$PACKAGE_DIRECTORY/files" + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$ORIGINAL_MANIFEST") + test "$MANIFEST_DIGEST" != "null" + jq -e \ + --arg source "$EXPECTED_COMMIT" \ + --arg digest "$MANIFEST_DIGEST" \ + --slurpfile manifest "$ORIGINAL_MANIFEST" \ + '.schema_version == "1" and + .source_commit == $source and + .manifest_digest == $digest and + (.has_changes | type == "boolean") and + if .has_changes then + (.retained_candidates | length) > 0 and + ([.retained_candidates[].candidate_id] | length) == + ([.retained_candidates[].candidate_id] | unique | length) and + all(.retained_candidates[]; + . as $retained | + any($manifest[0].candidates[]; + .candidate_id == $retained.candidate_id and + .path == $retained.path and + .decision.eligible == true and + ((.owner.test_fqns | sort) == ($retained.test_fqns | sort)))) and + (.changed_files | length) > 0 and + ([.changed_files[].path] | length) == + ([.changed_files[].path] | unique | length) and + ([.changed_files[].path] | sort) == (.changed_paths | sort) and + ([.retained_candidates[].path] | unique | sort) == (.changed_paths | sort) and + all(.changed_files[]; + (.path | type == "string") and + (.content_sha256 | type == "string" and test("^[0-9a-f]{64}$"))) and + (.pr_title | type == "string" and + startswith("[unskip-closed-tests] ") and length <= 256) and + (.pr_body | type == "string" and + startswith("")) + else + (.retained_candidates | length) == 0 and + (.changed_files | length) == 0 and + (.changed_paths | length) == 0 and + .pr_title == "" and + .pr_body == "" + end' \ + "$RESULT_PATH" >/dev/null + + if [ "$(jq -r '.has_changes' "$RESULT_PATH")" = "false" ]; then + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -print -quit)" + git diff --quiet + echo "::notice::No selected candidate passed verification." + exit 0 + fi + + DEFAULT_BRANCH=$(gh api "repos/${EXPECTED_REPOSITORY}" --jq '.default_branch') + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + test "$CURRENT_HEAD" = "$EXPECTED_COMMIT" || + { echo "::notice::Default branch advanced; leaving verified changes unpublished."; exit 0; } + + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + test "$EXISTING" -eq 0 || + { echo "::notice::An unskip pull request is already open."; exit 0; } + + TITLE=$(jq -r '.pr_title' "$RESULT_PATH") + BODY_FILE="$RUNNER_TEMP/unskip-closed-tests-pr-body.md" + EXPECTED_FILES="$RUNNER_TEMP/unskip-closed-tests-expected-files.bin" + EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.bin" + ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.bin" + EXPECTED_BLOBS="$RUNNER_TEMP/unskip-closed-tests-expected-blobs.txt" + ACTUAL_BLOBS="$RUNNER_TEMP/unskip-closed-tests-actual-blobs.txt" + jq -r '.pr_body' "$RESULT_PATH" > "$BODY_FILE" + jq -j '.changed_files[] | .path, "\u0000", .content_sha256, "\u0000"' \ + "$RESULT_PATH" > "$EXPECTED_FILES" + jq -j '.changed_files[].path, "\u0000"' "$RESULT_PATH" | sort -z > "$EXPECTED_PATHS" + jq -r '[.changed_files[].content_sha256] | unique | sort | .[]' \ + "$RESULT_PATH" > "$EXPECTED_BLOBS" + find "$PACKAGE_DIRECTORY/files" -mindepth 1 -maxdepth 1 -type f \ + -printf '%f\n' | sort > "$ACTUAL_BLOBS" + cmp "$EXPECTED_BLOBS" "$ACTUAL_BLOBS" + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -not -type f -print -quit)" + + git diff --cached --quiet + while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do + test -n "$path" + test "${path#/}" = "$path" + case "/$path/" in + *"/../"*|*"/./"*) exit 20 ;; + esac + case "$path" in + *.cs) ;; + *) echo "::error::Unexpected changed path: $path"; exit 20 ;; + esac + git ls-files --error-unmatch -- "$path" >/dev/null + test -f "$path" + test ! -L "$path" + SOURCE_SHA=$(jq -er \ + --arg path "$path" \ + '[.candidates[] | select(.path == $path) | .source_sha256] | + unique | select(length == 1) | .[0]' \ + "$ORIGINAL_MANIFEST") + ACTUAL_SOURCE_SHA=$(sha256sum -- "$path") + ACTUAL_SOURCE_SHA=${ACTUAL_SOURCE_SHA%% *} + test "$ACTUAL_SOURCE_SHA" = "$SOURCE_SHA" || + { echo "::error::Source content changed for $path"; exit 20; } + BLOB="$PACKAGE_DIRECTORY/files/$expected_sha" + test -f "$BLOB" + test ! -L "$BLOB" + ACTUAL_BLOB_SHA=$(sha256sum -- "$BLOB") + ACTUAL_BLOB_SHA=${ACTUAL_BLOB_SHA%% *} + test "$ACTUAL_BLOB_SHA" = "$expected_sha" || + { echo "::error::Verified package content changed for $path"; exit 20; } + cp -- "$BLOB" "$path" + git add -- "$path" + done < "$EXPECTED_FILES" + + git diff --quiet + git diff --cached --name-only --no-renames -z | sort -z > "$ACTUAL_PATHS" + cmp "$EXPECTED_PATHS" "$ACTUAL_PATHS" + while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do + staged_sha=$(git show ":$path" | sha256sum) + staged_sha=${staged_sha%% *} + test "$staged_sha" = "$expected_sha" || + { echo "::error::Staged content does not match verified content for $path"; exit 20; } + done < "$EXPECTED_FILES" + + rm -rf .github/workflows/unskip-closed-tests-tool/bin \ + .github/workflows/unskip-closed-tests-tool/obj + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Re-enable tests with resolved tracking items" + + BRANCH="automation/unskip-closed-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + git push origin "HEAD:refs/heads/$BRANCH" + + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + if [ "$CURRENT_HEAD" != "$EXPECTED_COMMIT" ]; then + git push origin --delete "$BRANCH" + echo "::notice::Default branch advanced before PR creation; removed the unpublished branch." + exit 0 + fi + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + if [ "$EXISTING" -ne 0 ]; then + git push origin --delete "$BRANCH" + echo "::notice::Another unskip pull request opened; removed the duplicate branch." + exit 0 + fi + + PR_URL=$(gh pr create \ + --repo "$EXPECTED_REPOSITORY" \ + --base "$DEFAULT_BRANCH" \ + --head "$BRANCH" \ + --draft \ + --title "$TITLE" \ + --body-file "$BODY_FILE") + + LIVE=$(gh pr view "$PR_URL" \ + --repo "$EXPECTED_REPOSITORY" \ + --json baseRefName,baseRefOid,body,isDraft,headRefName,number,state,url) + test "$(printf '%s' "$LIVE" | jq -r '.isDraft')" = "true" + test "$(printf '%s' "$LIVE" | jq -r '.headRefName')" = "$BRANCH" + test "$(printf '%s' "$LIVE" | jq -r '.baseRefName')" = "$DEFAULT_BRANCH" + printf '%s' "$LIVE" | jq -r '.body' | grep -F '"); + body.AppendLine(); + body.AppendLine("## Verified unskips"); + body.AppendLine(); + foreach (Candidate candidate in retained) + { + body.Append("- `"); + body.Append(candidate.Path); + body.Append("` β€” "); + body.Append(string.Join(", ", candidate.Owner.TestFqns.Select(static fqn => $"`{fqn}`"))); + body.Append(" ("); + body.Append(string.Join(", ", candidate.CanonicalIssueReferences.Select(static reference => + $"[{reference.Canonical}]({reference.Url})"))); + body.AppendLine(")"); + } + + body.AppendLine(); + body.AppendLine("Each retained edit was verified independently by the configured trusted command and exact TRX FQN mapping."); + if (reverted.Count > 0) + { + body.AppendLine(); + body.AppendLine($"The helper reverted {reverted.Count} candidate(s) that did not satisfy verification."); + } + + return body.ToString().TrimEnd(); + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/AssemblyInfo.cs b/.github/workflows/unskip-closed-tests-tool/AssemblyInfo.cs new file mode 100644 index 0000000000..7826586c06 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/AssemblyInfo.cs @@ -0,0 +1,3 @@ +ο»Ώusing System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("UnskipClosedTests.Tool.Tests")] diff --git a/.github/workflows/unskip-closed-tests-tool/ConfigLoader.cs b/.github/workflows/unskip-closed-tests-tool/ConfigLoader.cs new file mode 100644 index 0000000000..da4c971b97 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/ConfigLoader.cs @@ -0,0 +1,206 @@ +ο»Ώusing System.Text.Json; + +namespace UnskipClosedTests.Tool; + +internal static class ConfigLoader +{ + private static readonly HashSet AllowedProperties = + [ + "schema_version", + "source_roots", + "excluded_globs", + "generated_globs", + "ignore_attribute_names", + "test_attribute_names", + "verification", + ]; + + public static ToolConfig Load(string path) + { + using JsonDocument document = JsonSupport.ReadDocument(path); + JsonElement root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + throw new ContractException("Config root must be an object."); + } + + RejectUnknownProperties(root, AllowedProperties, "config"); + ToolConfig config = new() + { + SchemaVersion = RequiredString(root, "schema_version"), + SourceRoots = RequiredStringArray(root, "source_roots"), + ExcludedGlobs = OptionalStringArray(root, "excluded_globs"), + GeneratedGlobs = OptionalStringArray(root, "generated_globs"), + IgnoreAttributeNames = RequiredStringArray(root, "ignore_attribute_names"), + TestAttributeNames = RequiredStringArray(root, "test_attribute_names"), + }; + + if (!root.TryGetProperty("verification", out JsonElement verification) || + verification.ValueKind != JsonValueKind.Object) + { + throw new ContractException("verification must be an object."); + } + + RejectUnknownProperties(verification, ["command", "timeout_seconds"], "verification"); + config.VerificationCommand = RequiredStringArray(verification, "command"); + config.VerificationTimeoutSeconds = RequiredPositiveInt(verification, "timeout_seconds"); + + Validate(config); + return config; + } + + public static string Digest(ToolConfig config) => JsonSupport.CanonicalDigest(config); + + private static void Validate(ToolConfig config) + { + if (config.SchemaVersion != "1") + { + throw new ContractException($"Unsupported config schema_version '{config.SchemaVersion}'."); + } + + if (config.SourceRoots.Count == 0) + { + throw new ContractException("source_roots must contain at least one path."); + } + + if (config.IgnoreAttributeNames.Count == 0 || config.TestAttributeNames.Count == 0) + { + throw new ContractException("ignore_attribute_names and test_attribute_names must not be empty."); + } + + if (config.VerificationCommand.Count == 0) + { + throw new ContractException("verification.command must not be empty."); + } + + ValidateUniqueNonEmpty(config.SourceRoots, "source_roots"); + ValidateUniqueNonEmpty(config.ExcludedGlobs, "excluded_globs"); + ValidateUniqueNonEmpty(config.GeneratedGlobs, "generated_globs"); + ValidateUniqueNonEmpty(config.IgnoreAttributeNames, "ignore_attribute_names"); + ValidateUniqueNonEmpty(config.TestAttributeNames, "test_attribute_names"); + ValidateUniqueNonEmpty(config.VerificationCommand, "verification.command", requireUnique: false); + + foreach (string root in config.SourceRoots) + { + PathRules.ValidateRelativePath(root, "source root"); + } + + foreach (string glob in config.ExcludedGlobs.Concat(config.GeneratedGlobs)) + { + if (Path.IsPathRooted(glob) || glob.Contains('\\')) + { + throw new ContractException($"Glob '{glob}' must be repository-relative and use '/' separators."); + } + + if (glob.Split('/').Any(static segment => segment == "..")) + { + throw new ContractException($"Glob '{glob}' contains traversal."); + } + } + + foreach (string name in config.IgnoreAttributeNames.Concat(config.TestAttributeNames)) + { + if (!IsAttributeName(name)) + { + throw new ContractException($"Attribute name '{name}' is not a simple or qualified C# identifier."); + } + } + } + + private static bool IsAttributeName(string value) + { + string[] pieces = value.Split('.'); + return pieces.Length > 0 && pieces.All(static piece => + piece.Length > 0 && + (char.IsLetter(piece[0]) || piece[0] == '_') && + piece.Skip(1).All(static character => char.IsLetterOrDigit(character) || character == '_')); + } + + private static void ValidateUniqueNonEmpty(List values, string name, bool requireUnique = true) + { + if (values.Any(static value => string.IsNullOrWhiteSpace(value))) + { + throw new ContractException($"{name} contains an empty value."); + } + + if (requireUnique && values.Distinct(StringComparer.Ordinal).Count() != values.Count) + { + throw new ContractException($"{name} contains duplicate values."); + } + } + + private static void RejectUnknownProperties(JsonElement element, HashSet allowed, string context) + { + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + throw new ContractException($"Unknown {context} property '{property.Name}'."); + } + } + } + + private static string RequiredString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind != JsonValueKind.String) + { + throw new ContractException($"{name} must be a string."); + } + + return value.GetString()!; + } + + private static List RequiredStringArray(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value)) + { + throw new ContractException($"{name} is required."); + } + + return ReadStringArray(value, name); + } + + private static List OptionalStringArray(JsonElement element, string name) => + element.TryGetProperty(name, out JsonElement value) ? ReadStringArray(value, name) : []; + + private static List ReadStringArray(JsonElement value, string name) + { + if (value.ValueKind != JsonValueKind.Array) + { + throw new ContractException($"{name} must be an array."); + } + + List result = []; + foreach (JsonElement item in value.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + throw new ContractException($"{name} must contain only strings."); + } + + result.Add(item.GetString()!); + } + + return result; + } + + private static int RequiredPositiveInt(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value)) + { + throw new ContractException($"{name} is required."); + } + + return ReadPositiveInt(value, name); + } + + private static int ReadPositiveInt(JsonElement value, string name) + { + if (value.ValueKind != JsonValueKind.Number || !value.TryGetInt32(out int result) || result <= 0) + { + throw new ContractException($"{name} must be a positive 32-bit integer."); + } + + return result; + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Directory.Build.props b/.github/workflows/unskip-closed-tests-tool/Directory.Build.props new file mode 100644 index 0000000000..058246e408 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Directory.Build.props @@ -0,0 +1 @@ + diff --git a/.github/workflows/unskip-closed-tests-tool/Directory.Build.targets b/.github/workflows/unskip-closed-tests-tool/Directory.Build.targets new file mode 100644 index 0000000000..058246e408 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Directory.Build.targets @@ -0,0 +1 @@ + diff --git a/.github/workflows/unskip-closed-tests-tool/Directory.Packages.props b/.github/workflows/unskip-closed-tests-tool/Directory.Packages.props new file mode 100644 index 0000000000..5f9708a97f --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Directory.Packages.props @@ -0,0 +1,5 @@ + + + false + + diff --git a/.github/workflows/unskip-closed-tests-tool/GitRepository.cs b/.github/workflows/unskip-closed-tests-tool/GitRepository.cs new file mode 100644 index 0000000000..8922c5a0a4 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/GitRepository.cs @@ -0,0 +1,168 @@ +ο»Ώusing System.Diagnostics; +using System.Text; +using System.Text.RegularExpressions; + +namespace UnskipClosedTests.Tool; + +internal sealed class GitRepository +{ + private static readonly Regex GitHubRemotePattern = new( + @"(?:github\.com[:/])(?[^/:\s]+)/(?[^/\s]+?)(?:\.git)?$", + RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking); + + private GitRepository(string root, string commit, string objectFormat, string repository) + { + Root = root; + Commit = commit; + ObjectFormat = objectFormat; + Repository = repository; + } + + public string Root { get; } + public string Commit { get; } + public string ObjectFormat { get; } + public string Repository { get; } + + public static GitRepository Open(string requestedRoot, string? repositoryOverride) + { + string root = RunGit(requestedRoot, ["rev-parse", "--show-toplevel"]).Trim(); + if (root.Length == 0) + { + throw new ContractException("Could not determine the repository root."); + } + + root = Path.GetFullPath(root); + string requested = Path.GetFullPath(requestedRoot); + if (!string.Equals(root, requested, StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException($"--repo-root must be the exact git repository root '{root}'."); + } + + string commit = RunGit(root, ["rev-parse", "HEAD"]).Trim(); + string objectFormat = RunGit(root, ["rev-parse", "--show-object-format"]).Trim(); + if (objectFormat is not ("sha1" or "sha256")) + { + throw new ContractException($"Unsupported git object format '{objectFormat}'."); + } + + string repository = repositoryOverride is null + ? InferRepository(root) + : ValidateRepository(repositoryOverride); + return new GitRepository(root, commit, objectFormat, repository); + } + + public string HeadBlobOid(string path) + { + string normalized = PathRules.ValidateRelativePath(path, "source path"); + string output = RunGit(Root, ["ls-tree", Commit, "--", normalized]).Trim(); + if (output.Length == 0) + { + throw new ContractException($"Source path '{normalized}' is not tracked at commit {Commit}."); + } + + string[] tabParts = output.Split('\t'); + string[] metadata = tabParts[0].Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (metadata.Length != 3 || metadata[1] != "blob") + { + throw new ContractException($"Source path '{normalized}' is not a regular tracked blob."); + } + + if (metadata[0] == "120000") + { + throw new ContractException($"Source path '{normalized}' is a git symlink."); + } + + return metadata[2]; + } + + public byte[] HeadBytes(string path) + { + string normalized = PathRules.ValidateRelativePath(path, "source path"); + return RunGitBytes(Root, ["show", $"{Commit}:{normalized}"]); + } + + public void RequireWorktreeMatchesHead(string path, byte[] bytes) + { + _ = bytes; + string normalized = PathRules.ValidateRelativePath(path, "source path"); + string status = RunGit( + Root, + ["status", "--porcelain=v1", "--untracked-files=no", "--", normalized]).Trim(); + if (status.Length != 0) + { + throw new ContractException($"Source path '{path}' does not match checked-out commit {Commit}."); + } + } + + public string MetadataDirectory() + { + string value = RunGit(Root, ["rev-parse", "--git-dir"]).Trim(); + return Path.GetFullPath(Path.IsPathRooted(value) ? value : Path.Combine(Root, value)); + } + + private static string InferRepository(string root) + { + string remote = RunGit(root, ["config", "--get", "remote.origin.url"], allowFailure: true).Trim(); + Match match = GitHubRemotePattern.Match(remote); + if (!match.Success) + { + throw new ContractException("Could not infer owner/repo from remote.origin.url; pass --repository."); + } + + return ValidateRepository($"{match.Groups["owner"].Value}/{match.Groups["repo"].Value}"); + } + + private static string ValidateRepository(string repository) + { + string[] parts = repository.Split('/'); + if (parts.Length != 2 || parts.Any(static part => part.Length == 0 || part is "." or "..")) + { + throw new ContractException($"Repository '{repository}' must be owner/repo."); + } + + return $"{parts[0].ToLowerInvariant()}/{parts[1].ToLowerInvariant()}"; + } + + private static string RunGit(string workingDirectory, IReadOnlyList arguments, bool allowFailure = false) => + Encoding.UTF8.GetString(RunGitBytes(workingDirectory, arguments, allowFailure)); + + private static byte[] RunGitBytes(string workingDirectory, IReadOnlyList arguments, bool allowFailure = false) + { + ProcessStartInfo startInfo = new("git") + { + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (string argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + try + { + using Process process = Process.Start(startInfo) + ?? throw new InfrastructureException("Could not start git."); + using MemoryStream output = new(); + process.StandardOutput.BaseStream.CopyTo(output); + string error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (process.ExitCode != 0 && !allowFailure) + { + throw new ContractException($"git {string.Join(' ', arguments)} failed: {error.Trim()}"); + } + + return output.ToArray(); + } + catch (ContractException) + { + throw; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidOperationException) + { + throw new InfrastructureException("Could not invoke git.", ex); + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/InventoryEngine.cs b/.github/workflows/unskip-closed-tests-tool/InventoryEngine.cs new file mode 100644 index 0000000000..dda584dc6c --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/InventoryEngine.cs @@ -0,0 +1,587 @@ +ο»Ώusing System.Text; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; + +namespace UnskipClosedTests.Tool; + +internal static partial class InventoryEngine +{ + private sealed record ParsedFile( + string Path, + string FullPath, + byte[] Bytes, + string BlobOid, + SyntaxTree Tree, + CompilationUnitSyntax Root); + + private sealed record PendingCandidate( + ParsedFile File, + AttributeSyntax Attribute, + OwnerIdentity Owner, + string StableOwnerId, + List References, + List StructuralDeferrals); + + public static Manifest Create(string requestedRoot, string? repositoryOverride, ToolConfig config) + { + GitRepository repository = GitRepository.Open(requestedRoot, repositoryOverride); + List files = LoadFiles(repository, config); + Dictionary typeDeclarationCounts = CountTypeDeclarations(files); + List pending = []; + + foreach (ParsedFile file in files) + { + foreach (AttributeSyntax attribute in file.Root.DescendantNodes().OfType()) + { + if (!AttributeMatches(attribute, config.IgnoreAttributeNames)) + { + continue; + } + + List references = ExtractReferences(attribute, repository.Repository); + if (references.Count == 0) + { + continue; + } + + if (attribute.FirstAncestorOrSelf() is MethodDeclarationSyntax method && + method.AttributeLists.Any(list => list.Span.Contains(attribute.Span))) + { + OwnerIdentity owner = CreateMethodOwner(method, config); + List deferrals = []; + if (owner.TestFqns.Count == 0) + { + deferrals.Add("no_enumerated_tests"); + } + if (HasGeneratedMarker(method)) + { + deferrals.Add("generated_declaration"); + } + + string stableOwnerId = StableOwnerId(repository.Repository, file.Path, owner, method); + pending.Add(new PendingCandidate(file, attribute, owner, stableOwnerId, references, deferrals)); + continue; + } + + if (attribute.FirstAncestorOrSelf() is ClassDeclarationSyntax type && + type.AttributeLists.Any(list => list.Span.Contains(attribute.Span))) + { + (OwnerIdentity owner, List deferrals) = + CreateClassOwner(type, config, typeDeclarationCounts); + if (HasGeneratedMarker(type)) + { + deferrals.Add("generated_declaration"); + } + + string stableOwnerId = StableOwnerId(repository.Repository, file.Path, owner, type); + pending.Add(new PendingCandidate(file, attribute, owner, stableOwnerId, references, deferrals)); + } + } + } + + List candidates = []; + foreach (IGrouping ownerGroup in pending + .OrderBy(static item => item.File.Path, StringComparer.Ordinal) + .ThenBy(static item => item.Attribute.SpanStart) + .GroupBy(static item => item.StableOwnerId, StringComparer.Ordinal)) + { + int ordinal = 0; + foreach (PendingCandidate item in ownerGroup) + { + ordinal++; + FileLinePositionSpan lineSpan = item.Attribute.GetLocation().GetLineSpan(); + SourceSpan sourceSpan = new() + { + Start = item.Attribute.SpanStart, + Length = item.Attribute.Span.Length, + StartLine = lineSpan.StartLinePosition.Line + 1, + StartColumn = lineSpan.StartLinePosition.Character + 1, + EndLine = lineSpan.EndLinePosition.Line + 1, + EndColumn = lineSpan.EndLinePosition.Character + 1, + }; + string attributeText = item.File.Root.SyntaxTree.GetText().ToString(item.Attribute.Span); + string candidateId = JsonSupport.Sha256( + $"candidate-v1\0{repository.Repository}\0{item.File.Path}\0{item.StableOwnerId}\0" + + $"{item.File.BlobOid}\0{sourceSpan.Start}:{sourceSpan.Length}\0{ordinal}"); + + candidates.Add(new Candidate + { + CandidateId = candidateId, + StableOwnerId = item.StableOwnerId, + Path = item.File.Path, + BlobOid = item.File.BlobOid, + SourceSha256 = JsonSupport.Sha256(item.File.Bytes), + AttributeSpan = sourceSpan, + AttributeTextSha256 = JsonSupport.Sha256(attributeText), + Owner = item.Owner, + CanonicalIssueReferences = item.References, + Decision = new CandidateDecision + { + Eligible = false, + Deferrals = item.StructuralDeferrals.Order(StringComparer.Ordinal).ToList(), + }, + }); + } + } + + candidates = candidates + .OrderBy(static candidate => candidate.Path, StringComparer.Ordinal) + .ThenBy(static candidate => candidate.AttributeSpan.Start) + .ToList(); + Manifest manifest = new() + { + Repository = repository.Repository, + SourceCommit = repository.Commit, + GitObjectFormat = repository.ObjectFormat, + ConfigDigest = ConfigLoader.Digest(config), + CandidateCount = candidates.Count, + Candidates = candidates, + }; + manifest.ManifestDigest = JsonSupport.ManifestDigest(manifest); + return manifest; + } + + private static List LoadFiles(GitRepository repository, ToolConfig config) + { + Dictionary paths = new(StringComparer.Ordinal); + foreach (string configuredRoot in config.SourceRoots) + { + string normalizedRoot = PathRules.ValidateRelativePath(configuredRoot, "source root"); + string fullRoot = PathRules.ResolveInsideRoot(repository.Root, normalizedRoot, "source root"); + if (File.Exists(fullRoot)) + { + if (!normalizedRoot.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException($"Source root '{normalizedRoot}' is not a C# file."); + } + + if (PathRules.MatchesAnyGlob(normalizedRoot, config.ExcludedGlobs.Concat(config.GeneratedGlobs))) + { + throw new ContractException($"Explicit source root '{normalizedRoot}' is excluded or generated."); + } + + paths[normalizedRoot] = fullRoot; + continue; + } + + if (!Directory.Exists(fullRoot)) + { + continue; + } + + PathRules.RejectReparsePoints(repository.Root, fullRoot); + foreach (string file in Directory.EnumerateFiles(fullRoot, "*", SearchOption.AllDirectories)) + { + string extension = Path.GetExtension(file); + if (!string.Equals(extension, ".cs", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + string relative = Path.GetRelativePath(repository.Root, file).Replace('\\', '/'); + relative = PathRules.ValidateRelativePath(relative, "source path"); + if (PathRules.MatchesAnyGlob(relative, config.ExcludedGlobs.Concat(config.GeneratedGlobs))) + { + continue; + } + + paths[relative] = file; + } + } + + List result = []; + foreach ((string relative, string fullPath) in paths.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + PathRules.RejectReparsePoints(repository.Root, fullPath); + byte[] bytes; + try + { + bytes = File.ReadAllBytes(fullPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + throw new InfrastructureException($"Could not read source path '{relative}'.", ex); + } + + string blobOid = repository.HeadBlobOid(relative); + repository.RequireWorktreeMatchesHead(relative, bytes); + string source; + try + { + int offset = bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF ? 3 : 0; + source = new UTF8Encoding(false, true).GetString(bytes, offset, bytes.Length - offset); + } + catch (DecoderFallbackException ex) + { + throw new ContractException($"Source path '{relative}' is not valid UTF-8: {ex.Message}"); + } + + string prefix = source[..Math.Min(source.Length, 2048)]; + if (prefix.Contains(" errors = tree.GetDiagnostics() + .Where(static diagnostic => diagnostic.Severity == DiagnosticSeverity.Error) + .ToList(); + if (errors.Count > 0) + { + throw new ContractException( + $"Source path '{relative}' has C# parse errors: {string.Join("; ", errors.Take(3))}"); + } + + result.Add(new ParsedFile(relative, fullPath, bytes, blobOid, tree, tree.GetCompilationUnitRoot())); + } + + return result; + } + + private static Dictionary CountTypeDeclarations(IEnumerable files) + { + Dictionary counts = new(StringComparer.Ordinal); + foreach (TypeDeclarationSyntax declaration in files.SelectMany(static file => + file.Root.DescendantNodes().OfType())) + { + string fqn = TypeFqn(declaration); + counts[fqn] = counts.GetValueOrDefault(fqn) + 1; + } + + return counts; + } + + private static OwnerIdentity CreateMethodOwner(MethodDeclarationSyntax method, ToolConfig config) + { + TypeDeclarationSyntax? type = method.Ancestors().OfType().FirstOrDefault(); + if (type is null) + { + throw new ContractException("An Ignore attribute on a method has no actual containing type."); + } + + string typeFqn = TypeFqn(type); + string signature = MethodSignature(method); + bool isTest = method.AttributeLists.SelectMany(static list => list.Attributes) + .Any(attribute => AttributeMatches(attribute, config.TestAttributeNames)); + return new OwnerIdentity + { + Kind = "method", + Namespace = NamespaceName(type), + ContainingTypes = ContainingTypeNames(type), + TypeFqn = typeFqn, + DeclarationId = MethodDeclarationId(method), + MethodName = method.Identifier.ValueText, + MethodSignature = signature, + TestFqns = isTest ? [$"{typeFqn}.{method.Identifier.ValueText}"] : [], + }; + } + + private static (OwnerIdentity Owner, List Deferrals) CreateClassOwner( + ClassDeclarationSyntax type, + ToolConfig config, + IReadOnlyDictionary declarationCounts) + { + string typeFqn = TypeFqn(type); + List deferrals = []; + List containingTypes = ContainingTypeNames(type); + if (containingTypes.Count > 1) + { + deferrals.Add("class_is_nested"); + } + + if (type.Modifiers.Any(SyntaxKind.PartialKeyword)) + { + deferrals.Add("class_is_partial"); + } + + if (type.BaseList is not null && type.BaseList.Types.Count > 0) + { + deferrals.Add("class_has_base_types"); + } + + if (declarationCounts.GetValueOrDefault(typeFqn) != 1) + { + deferrals.Add("duplicate_type_declarations"); + } + + List tests = type.Members + .OfType() + .Where(method => method.AttributeLists.SelectMany(static list => list.Attributes) + .Any(attribute => AttributeMatches(attribute, config.TestAttributeNames))) + .Select(method => $"{typeFqn}.{method.Identifier.ValueText}") + .Order(StringComparer.Ordinal) + .ToList(); + if (tests.Count == 0) + { + deferrals.Add("no_enumerated_tests"); + } + + if (tests.Distinct(StringComparer.Ordinal).Count() != tests.Count) + { + deferrals.Add("ambiguous_test_fqns"); + } + + OwnerIdentity owner = new() + { + Kind = "class", + Namespace = NamespaceName(type), + ContainingTypes = containingTypes, + TypeFqn = typeFqn, + DeclarationId = $"T:{typeFqn}", + TestFqns = tests.Distinct(StringComparer.Ordinal).ToList(), + }; + return (owner, deferrals.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList()); + } + + private static string StableOwnerId( + string repository, + string path, + OwnerIdentity owner, + MemberDeclarationSyntax declaration) + { + int declarationOrdinal = declaration switch + { + MethodDeclarationSyntax method => method.SyntaxTree.GetRoot() + .DescendantNodes() + .OfType() + .Where(candidate => string.Equals( + MethodDeclarationId(candidate), + owner.DeclarationId, + StringComparison.Ordinal)) + .Count(candidate => candidate.SpanStart < method.SpanStart) + 1, + TypeDeclarationSyntax type => type.SyntaxTree.GetRoot() + .DescendantNodes() + .OfType() + .Where(candidate => string.Equals( + $"T:{TypeFqn(candidate)}", + owner.DeclarationId, + StringComparison.Ordinal)) + .Count(candidate => candidate.SpanStart < type.SpanStart) + 1, + _ => throw new ContractException("Unsupported owner declaration kind."), + }; + return JsonSupport.Sha256( + $"owner-v1\0{repository}\0{path}\0{owner.DeclarationId}\0{declarationOrdinal}"); + } + + private static bool HasGeneratedMarker(MemberDeclarationSyntax declaration) => + HasDirectGeneratedMarker(declaration) || + declaration.Ancestors().OfType().Any(HasDirectGeneratedMarker); + + private static bool HasDirectGeneratedMarker(MemberDeclarationSyntax declaration) => + declaration.AttributeLists + .SelectMany(static list => list.Attributes) + .Any(static attribute => + { + string name = attribute.Name.WithoutTrivia().ToFullString() + .Replace("global::", "", StringComparison.Ordinal) + .Split('.') + .Last(); + if (name.EndsWith("Attribute", StringComparison.Ordinal)) + { + name = name[..^"Attribute".Length]; + } + + return name is "GeneratedCode" or "CompilerGenerated"; + }); + + private static string MethodSignature(MethodDeclarationSyntax method) + { + string explicitInterface = method.ExplicitInterfaceSpecifier is null + ? "" + : $"{method.ExplicitInterfaceSpecifier.Name.WithoutTrivia().ToFullString()}."; + string arity = method.TypeParameterList is null ? "" : $"`{method.TypeParameterList.Parameters.Count}"; + string parameters = string.Join(",", + method.ParameterList.Parameters.Select(static parameter => + $"{parameter.Modifiers.ToFullString().Trim()}:{parameter.Type?.WithoutTrivia().ToFullString() ?? "?"}")); + return $"{explicitInterface}{method.Identifier.ValueText}{arity}({parameters})"; + } + + private static string MethodDeclarationId(MethodDeclarationSyntax method) + { + TypeDeclarationSyntax? type = method.Ancestors().OfType().FirstOrDefault(); + if (type is null) + { + throw new ContractException("A method owner has no actual containing type."); + } + + return $"M:{TypeFqn(type)}.{MethodSignature(method)}"; + } + + private static string TypeFqn(TypeDeclarationSyntax type) + { + List parts = []; + string namespaceName = NamespaceName(type); + if (namespaceName.Length > 0) + { + parts.Add(namespaceName); + } + + parts.AddRange(ContainingTypeNames(type)); + return string.Join('.', parts); + } + + private static string NamespaceName(SyntaxNode node) => + string.Join('.', + node.Ancestors() + .OfType() + .Reverse() + .Select(static declaration => declaration.Name.WithoutTrivia().ToFullString())); + + private static List ContainingTypeNames(TypeDeclarationSyntax type) => + type.AncestorsAndSelf() + .OfType() + .Reverse() + .Select(static declaration => + declaration.TypeParameterList is null + ? declaration.Identifier.ValueText + : $"{declaration.Identifier.ValueText}`{declaration.TypeParameterList.Parameters.Count}") + .ToList(); + + internal static bool AttributeMatches(AttributeSyntax attribute, IEnumerable configuredNames) + { + string actual = attribute.Name.WithoutTrivia().ToFullString().Replace("global::", "", StringComparison.Ordinal); + string actualShort = actual.Split('.').Last(); + return configuredNames.Any(configured => + { + string normalized = configured.EndsWith("Attribute", StringComparison.Ordinal) + ? configured[..^"Attribute".Length] + : configured; + string actualNormalized = actual.EndsWith("Attribute", StringComparison.Ordinal) + ? actual[..^"Attribute".Length] + : actual; + string shortNormalized = actualShort.EndsWith("Attribute", StringComparison.Ordinal) + ? actualShort[..^"Attribute".Length] + : actualShort; + return normalized.Contains('.', StringComparison.Ordinal) + ? string.Equals(normalized, actualNormalized, StringComparison.Ordinal) + : string.Equals(normalized, shortNormalized, StringComparison.Ordinal); + }); + } + + private static List ExtractReferences(AttributeSyntax attribute, string currentRepository) + { + List references = []; + if (attribute.ArgumentList is null) + { + return references; + } + + foreach (AttributeArgumentSyntax argument in attribute.ArgumentList.Arguments) + { + bool supportedName = argument.NameEquals is null || + string.Equals(argument.NameEquals.Name.Identifier.ValueText, "IgnoreMessage", StringComparison.Ordinal); + if (!supportedName || argument.NameColon is not null || + argument.Expression is not LiteralExpressionSyntax literal || + !literal.IsKind(SyntaxKind.StringLiteralExpression)) + { + continue; + } + + string value = literal.Token.ValueText; + references.AddRange(ParseReferences(value, currentRepository)); + } + + return references + .GroupBy(static reference => reference.Canonical, StringComparer.Ordinal) + .Select(static group => group.First()) + .OrderBy(static reference => reference.Canonical, StringComparer.Ordinal) + .ToList(); + } + + internal static IEnumerable ParseReferences(string value, string currentRepository) + { + List<(int Start, int Length, string Owner, string Repo, int Number, string Kind)> matches = []; + foreach (Match match in FullReferenceRegex().Matches(value)) + { + matches.Add(( + match.Index, + match.Length, + match.Groups["owner"].Value, + match.Groups["repo"].Value, + int.Parse(match.Groups["number"].Value, System.Globalization.CultureInfo.InvariantCulture), + match.Groups["kind"].Value.Equals("pull", StringComparison.OrdinalIgnoreCase) ? "pull_request" : "issue")); + } + + foreach (Match match in QualifiedReferenceRegex().Matches(value)) + { + if (matches.Any(existing => RangesOverlap(existing.Start, existing.Length, match.Index, match.Length))) + { + continue; + } + + matches.Add(( + match.Index, + match.Length, + match.Groups["owner"].Value, + match.Groups["repo"].Value, + int.Parse(match.Groups["number"].Value, System.Globalization.CultureInfo.InvariantCulture), + "unknown")); + } + + string[] current = currentRepository.Split('/'); + foreach (Match match in BareReferenceRegex().Matches(value)) + { + if (matches.Any(existing => RangesOverlap(existing.Start, existing.Length, match.Index, match.Length))) + { + continue; + } + + matches.Add(( + match.Index, + match.Length, + current[0], + current[1], + int.Parse(match.Groups["number"].Value, System.Globalization.CultureInfo.InvariantCulture), + "unknown")); + } + + foreach ((_, _, string ownerValue, string repoValue, int number, string kind) in matches.OrderBy(static item => item.Start)) + { + if (number <= 0) + { + continue; + } + + string owner = ownerValue.ToLowerInvariant(); + string repo = repoValue.ToLowerInvariant(); + string pathKind = kind == "pull_request" ? "pull" : "issues"; + yield return new IssueReference + { + Kind = kind, + Owner = owner, + Repo = repo, + Number = number, + Canonical = $"{owner}/{repo}#{number}", + Url = $"https://github.com/{owner}/{repo}/{pathKind}/{number}", + Eligibility = false, + State = "unknown", + StateReason = "unresolved", + }; + } + } + + private static bool RangesOverlap(int firstStart, int firstLength, int secondStart, int secondLength) => + firstStart < secondStart + secondLength && secondStart < firstStart + firstLength; + + [GeneratedRegex( + @"https://github\.com/(?[A-Za-z0-9_.-]+)/(?[A-Za-z0-9_.-]+)/(?issues|pull)/(?[1-9][0-9]*)(?![A-Za-z0-9_])", + RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)] + private static partial Regex FullReferenceRegex(); + + [GeneratedRegex( + @"(?[A-Za-z0-9_.-]+)/(?[A-Za-z0-9_.-]+)#(?[1-9][0-9]*)(?![A-Za-z0-9_])", + RegexOptions.CultureInvariant)] + private static partial Regex QualifiedReferenceRegex(); + + [GeneratedRegex( + @"(?[1-9][0-9]*)(?![A-Za-z0-9_])", + RegexOptions.CultureInvariant)] + private static partial Regex BareReferenceRegex(); +} diff --git a/.github/workflows/unskip-closed-tests-tool/IssueResolver.cs b/.github/workflows/unskip-closed-tests-tool/IssueResolver.cs new file mode 100644 index 0000000000..5281b1e624 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/IssueResolver.cs @@ -0,0 +1,483 @@ +ο»Ώusing System.Net; +using System.Net.Http.Headers; +using System.Text.Json; + +namespace UnskipClosedTests.Tool; + +internal static class IssueResolver +{ + private static readonly HashSet StructuralDeferrals = + [ + "no_enumerated_tests", + "class_is_nested", + "class_is_partial", + "class_has_base_types", + "duplicate_type_declarations", + "ambiguous_test_fqns", + "generated_declaration", + ]; + + public static async Task ResolveAsync(Manifest input, string? evidencePath) + { + ManifestValidator.Validate(input); + Manifest manifest = Clone(input); + IReferenceEvidenceProvider provider = evidencePath is null + ? new GitHubReferenceEvidenceProvider() + : FixtureReferenceEvidenceProvider.Load(evidencePath); + + Dictionary cache = new(StringComparer.Ordinal); + foreach (Candidate candidate in manifest.Candidates) + { + List deferrals = candidate.Decision.Deferrals + .Where(StructuralDeferrals.Contains) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToList(); + List resolvedReferences = []; + foreach (IssueReference reference in candidate.CanonicalIssueReferences) + { + if (!cache.TryGetValue(reference.Canonical, out EvidenceReference? evidence)) + { + evidence = await provider.GetAsync(reference); + cache.Add(reference.Canonical, evidence); + } + + IssueReference resolved = ResolveReference(reference, evidence); + resolvedReferences.Add(resolved); + if (!resolved.Eligibility) + { + deferrals.Add($"reference_not_eligible:{resolved.Canonical}:{resolved.StateReason}"); + } + } + + candidate.CanonicalIssueReferences = resolvedReferences + .OrderBy(static reference => reference.Canonical, StringComparer.Ordinal) + .ToList(); + if (candidate.Owner.TestFqns.Count == 0 && !deferrals.Contains("no_enumerated_tests", StringComparer.Ordinal)) + { + deferrals.Add("no_enumerated_tests"); + } + + candidate.Decision = new CandidateDecision + { + Eligible = deferrals.Count == 0 && + candidate.CanonicalIssueReferences.Count > 0 && + candidate.CanonicalIssueReferences.All(static reference => reference.Eligibility), + Deferrals = deferrals.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList(), + }; + } + + manifest.ManifestDigest = ""; + manifest.ManifestDigest = JsonSupport.ManifestDigest(manifest); + return manifest; + } + + private static IssueReference ResolveReference(IssueReference original, EvidenceReference evidence) + { + if (!string.Equals(evidence.Canonical, original.Canonical, StringComparison.Ordinal)) + { + throw new ContractException( + $"Evidence canonical '{evidence.Canonical}' does not match requested reference '{original.Canonical}'."); + } + + string kind = NormalizeKind(evidence.Kind.Length == 0 ? original.Kind : evidence.Kind); + if (!evidence.Accessible) + { + return Copy(original, kind, false, "inaccessible", "inaccessible", null); + } + + if (kind == "unknown") + { + return Copy(original, kind, false, "unknown", "unknown_reference_kind", null); + } + + string state = evidence.State.ToLowerInvariant(); + string stateReason = evidence.StateReason.ToLowerInvariant(); + if (kind == "issue") + { + bool eligible = state == "closed" && stateReason == "completed"; + string reason = eligible + ? "completed" + : state == "open" + ? "open" + : stateReason == "not_planned" + ? "not_planned" + : "not_completed"; + return Copy(original, kind, eligible, state, reason, null); + } + + bool merged = !string.IsNullOrWhiteSpace(evidence.MergedAt); + if (merged && !DateTimeOffset.TryParse( + evidence.MergedAt, + System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.RoundtripKind, + out _)) + { + throw new ContractException($"Pull request evidence '{evidence.Canonical}' has malformed merged_at."); + } + + return Copy( + original, + kind, + merged, + state.Length == 0 ? (merged ? "closed" : "unknown") : state, + merged ? "merged" : "not_merged", + evidence.MergedAt); + } + + private static IssueReference Copy( + IssueReference original, + string kind, + bool eligible, + string state, + string stateReason, + string? mergedAt) + { + string pathKind = kind == "pull_request" ? "pull" : "issues"; + return new IssueReference + { + Kind = kind, + Owner = original.Owner, + Repo = original.Repo, + Number = original.Number, + Canonical = original.Canonical, + Url = $"https://github.com/{original.Owner}/{original.Repo}/{pathKind}/{original.Number}", + Eligibility = eligible, + State = state, + StateReason = stateReason, + MergedAt = mergedAt, + }; + } + + private static string NormalizeKind(string kind) => kind.ToLowerInvariant() switch + { + "issue" => "issue", + "pull" or "pr" or "pull_request" => "pull_request", + "unknown" or "" => "unknown", + _ => throw new ContractException($"Unsupported evidence kind '{kind}'."), + }; + + private static Manifest Clone(Manifest input) + { + string json = JsonSerializer.Serialize(input, JsonSupport.Options); + return JsonSerializer.Deserialize(json, JsonSupport.Options) + ?? throw new InfrastructureException("Could not clone manifest."); + } + + private interface IReferenceEvidenceProvider + { + Task GetAsync(IssueReference reference); + } + + private sealed class FixtureReferenceEvidenceProvider( + IReadOnlyDictionary references) : IReferenceEvidenceProvider + { + public static FixtureReferenceEvidenceProvider Load(string path) + { + using JsonDocument document = JsonSupport.ReadDocument(path); + JsonElement root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + throw new ContractException("GitHub evidence root must be an object."); + } + + foreach (JsonProperty property in root.EnumerateObject()) + { + if (property.Name is not ("schema_version" or "references")) + { + throw new ContractException($"Unknown GitHub evidence property '{property.Name}'."); + } + } + + if (!root.TryGetProperty("schema_version", out JsonElement schema) || + schema.ValueKind != JsonValueKind.String || + schema.GetString() != "1") + { + throw new ContractException("GitHub evidence schema_version must be '1'."); + } + + if (!root.TryGetProperty("references", out JsonElement referencesElement)) + { + throw new ContractException("GitHub evidence references is required."); + } + + Dictionary references = new(StringComparer.Ordinal); + if (referencesElement.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement item in referencesElement.EnumerateArray()) + { + EvidenceReference evidence = ParseEvidence(item, null); + if (!references.TryAdd(evidence.Canonical, evidence)) + { + throw new ContractException($"Duplicate GitHub evidence '{evidence.Canonical}'."); + } + } + } + else if (referencesElement.ValueKind == JsonValueKind.Object) + { + foreach (JsonProperty property in referencesElement.EnumerateObject()) + { + EvidenceReference evidence = ParseEvidence(property.Value, property.Name); + if (!references.TryAdd(evidence.Canonical, evidence)) + { + throw new ContractException($"Duplicate GitHub evidence '{evidence.Canonical}'."); + } + } + } + else + { + throw new ContractException("GitHub evidence references must be an array or object."); + } + + return new FixtureReferenceEvidenceProvider(references); + } + + public Task GetAsync(IssueReference reference) + { + if (references.TryGetValue(reference.Canonical, out EvidenceReference? evidence)) + { + return Task.FromResult(evidence); + } + + return Task.FromResult(new EvidenceReference + { + Canonical = reference.Canonical, + Kind = reference.Kind, + Accessible = false, + State = "inaccessible", + StateReason = "inaccessible", + }); + } + + private static EvidenceReference ParseEvidence(JsonElement element, string? canonicalFromKey) + { + if (element.ValueKind != JsonValueKind.Object) + { + throw new ContractException("Each GitHub evidence entry must be an object."); + } + + HashSet allowed = ["canonical", "kind", "accessible", "state", "state_reason", "merged_at"]; + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + throw new ContractException($"Unknown GitHub evidence field '{property.Name}'."); + } + } + + string canonical = canonicalFromKey ?? RequiredString(element, "canonical"); + if (element.TryGetProperty("canonical", out JsonElement canonicalElement) && + (canonicalElement.ValueKind != JsonValueKind.String || + !string.Equals(canonicalElement.GetString(), canonical, StringComparison.Ordinal))) + { + throw new ContractException("GitHub evidence canonical key and field do not match."); + } + + string kind = RequiredString(element, "kind"); + bool accessible = true; + if (element.TryGetProperty("accessible", out JsonElement accessibleElement)) + { + if (accessibleElement.ValueKind is not (JsonValueKind.True or JsonValueKind.False)) + { + throw new ContractException("GitHub evidence accessible must be a boolean."); + } + + accessible = accessibleElement.GetBoolean(); + } + + string state = OptionalString(element, "state"); + string stateReason = OptionalNullableString(element, "state_reason") ?? ""; + string? mergedAt = OptionalNullableString(element, "merged_at"); + if (accessible && kind.Equals("issue", StringComparison.OrdinalIgnoreCase) && + state.Length == 0) + { + throw new ContractException($"Accessible issue evidence '{canonical}' requires state."); + } + + if (accessible && kind.Equals("issue", StringComparison.OrdinalIgnoreCase) && + state.Equals("closed", StringComparison.OrdinalIgnoreCase) && + stateReason.Length == 0) + { + throw new ContractException($"Closed issue evidence '{canonical}' requires state_reason."); + } + + if (accessible && NormalizeKind(kind) == "pull_request" && state.Length == 0) + { + throw new ContractException($"Accessible pull request evidence '{canonical}' requires state."); + } + + return new EvidenceReference + { + Canonical = canonical, + Kind = kind, + Accessible = accessible, + State = state, + StateReason = stateReason, + MergedAt = mergedAt, + }; + } + + private static string RequiredString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind != JsonValueKind.String) + { + throw new ContractException($"GitHub evidence {name} must be a string."); + } + + return value.GetString()!; + } + + private static string OptionalString(JsonElement element, string name) => + element.TryGetProperty(name, out JsonElement value) + ? value.ValueKind == JsonValueKind.String + ? value.GetString()! + : throw new ContractException($"GitHub evidence {name} must be a string.") + : ""; + + private static string? OptionalNullableString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind == JsonValueKind.Null) + { + return null; + } + + return value.ValueKind == JsonValueKind.String + ? value.GetString() + : throw new ContractException($"GitHub evidence {name} must be a string or null."); + } + } + + private sealed class GitHubReferenceEvidenceProvider : IReferenceEvidenceProvider + { + private readonly HttpClient _client; + + public GitHubReferenceEvidenceProvider() + { + string? token = Environment.GetEnvironmentVariable("GH_TOKEN"); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InfrastructureException("GH_TOKEN is required when --github-evidence is not supplied."); + } + + _client = new HttpClient + { + BaseAddress = new Uri("https://api.github.com/"), + Timeout = TimeSpan.FromSeconds(30), + }; + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + _client.DefaultRequestHeaders.UserAgent.ParseAdd("unskip-closed-tests-tool/1"); + _client.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github+json"); + _client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + } + + public async Task GetAsync(IssueReference reference) + { + try + { + using HttpResponseMessage issueResponse = await _client.GetAsync( + $"repos/{Uri.EscapeDataString(reference.Owner)}/{Uri.EscapeDataString(reference.Repo)}/issues/{reference.Number}"); + if (issueResponse.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) + { + if (issueResponse.Headers.TryGetValues("X-RateLimit-Remaining", out IEnumerable? remaining) && + remaining.Contains("0", StringComparer.Ordinal)) + { + throw new InfrastructureException("GitHub API rate limit was exhausted."); + } + + return Inaccessible(reference); + } + + if (issueResponse.StatusCode == HttpStatusCode.Unauthorized) + { + throw new InfrastructureException("GitHub authentication was rejected."); + } + + if (!issueResponse.IsSuccessStatusCode) + { + throw new InfrastructureException($"GitHub issue lookup returned {(int)issueResponse.StatusCode}."); + } + + using JsonDocument issue = JsonDocument.Parse(await issueResponse.Content.ReadAsStreamAsync()); + JsonElement issueRoot = issue.RootElement; + bool isPullRequest = issueRoot.TryGetProperty("pull_request", out _); + if (!isPullRequest) + { + return new EvidenceReference + { + Canonical = reference.Canonical, + Kind = "issue", + Accessible = true, + State = RequiredApiString(issueRoot, "state"), + StateReason = NullableApiString(issueRoot, "state_reason") ?? "", + }; + } + + using HttpResponseMessage pullResponse = await _client.GetAsync( + $"repos/{Uri.EscapeDataString(reference.Owner)}/{Uri.EscapeDataString(reference.Repo)}/pulls/{reference.Number}"); + if (!pullResponse.IsSuccessStatusCode) + { + if (pullResponse.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) + { + return Inaccessible(reference); + } + + throw new InfrastructureException($"GitHub pull request lookup returned {(int)pullResponse.StatusCode}."); + } + + using JsonDocument pull = JsonDocument.Parse(await pullResponse.Content.ReadAsStreamAsync()); + JsonElement pullRoot = pull.RootElement; + return new EvidenceReference + { + Canonical = reference.Canonical, + Kind = "pull_request", + Accessible = true, + State = RequiredApiString(pullRoot, "state"), + StateReason = "", + MergedAt = NullableApiString(pullRoot, "merged_at"), + }; + } + catch (InfrastructureException) + { + throw; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or IOException) + { + throw new InfrastructureException($"GitHub lookup failed for {reference.Canonical}.", ex); + } + } + + private static EvidenceReference Inaccessible(IssueReference reference) => new() + { + Canonical = reference.Canonical, + Kind = reference.Kind, + Accessible = false, + State = "inaccessible", + StateReason = "inaccessible", + }; + + private static string RequiredApiString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value) || value.ValueKind != JsonValueKind.String) + { + throw new InfrastructureException($"GitHub response omitted string field '{name}'."); + } + + return value.GetString()!; + } + + private static string? NullableApiString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement value)) + { + throw new InfrastructureException($"GitHub response omitted field '{name}'."); + } + + return value.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => value.GetString(), + _ => throw new InfrastructureException($"GitHub response field '{name}' is malformed."), + }; + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/JsonSupport.cs b/.github/workflows/unskip-closed-tests-tool/JsonSupport.cs new file mode 100644 index 0000000000..dce5ecf672 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/JsonSupport.cs @@ -0,0 +1,148 @@ +ο»Ώusing System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace UnskipClosedTests.Tool; + +internal static class JsonSupport +{ + public static readonly JsonSerializerOptions Options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DictionaryKeyPolicy = JsonNamingPolicy.SnakeCaseLower, + WriteIndented = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never, + }; + + public static T Read(string path) + { + try + { + string json = File.ReadAllText(path, Encoding.UTF8); + return JsonSerializer.Deserialize(json, Options) + ?? throw new ContractException($"JSON document '{path}' is empty."); + } + catch (ContractException) + { + throw; + } + catch (JsonException ex) + { + throw new ContractException($"Malformed JSON in '{path}': {ex.Message}"); + } + catch (IOException ex) + { + throw new InfrastructureException($"Could not read '{path}'.", ex); + } + catch (UnauthorizedAccessException ex) + { + throw new InfrastructureException($"Could not read '{path}'.", ex); + } + } + + public static JsonDocument ReadDocument(string path) + { + try + { + return JsonDocument.Parse(File.ReadAllBytes(path)); + } + catch (JsonException ex) + { + throw new ContractException($"Malformed JSON in '{path}': {ex.Message}"); + } + catch (IOException ex) + { + throw new InfrastructureException($"Could not read '{path}'.", ex); + } + } + + public static void Write(string path, T value) + { + try + { + string fullPath = Path.GetFullPath(path); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + File.WriteAllText(fullPath, JsonSerializer.Serialize(value, Options) + Environment.NewLine, new UTF8Encoding(false)); + } + catch (IOException ex) + { + throw new InfrastructureException($"Could not write '{path}'.", ex); + } + catch (UnauthorizedAccessException ex) + { + throw new InfrastructureException($"Could not write '{path}'.", ex); + } + } + + public static string Sha256(ReadOnlySpan bytes) => + Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + + public static string Sha256(string text) => Sha256(Encoding.UTF8.GetBytes(text)); + + public static string CanonicalDigest(T value, string? excludedProperty = null) + { + JsonNode node = JsonSerializer.SerializeToNode(value, Options) + ?? throw new InfrastructureException("Could not serialize a digest input."); + if (excludedProperty is not null && node is JsonObject root) + { + root.Remove(excludedProperty); + } + + StringBuilder builder = new(); + WriteCanonical(node, builder); + return Sha256(builder.ToString()); + } + + public static string ManifestDigest(Manifest manifest) => + CanonicalDigest(manifest, "manifest_digest"); + + private static void WriteCanonical(JsonNode? node, StringBuilder builder) + { + switch (node) + { + case null: + builder.Append("null"); + break; + case JsonObject obj: + builder.Append('{'); + bool firstProperty = true; + foreach ((string key, JsonNode? value) in obj.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + if (!firstProperty) + { + builder.Append(','); + } + + firstProperty = false; + builder.Append(JsonSerializer.Serialize(key)); + builder.Append(':'); + WriteCanonical(value, builder); + } + + builder.Append('}'); + break; + case JsonArray array: + builder.Append('['); + for (int i = 0; i < array.Count; i++) + { + if (i > 0) + { + builder.Append(','); + } + + WriteCanonical(array[i], builder); + } + + builder.Append(']'); + break; + case JsonValue value: + builder.Append(value.ToJsonString()); + break; + default: + throw new InfrastructureException("Unsupported JSON node while computing a digest."); + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/ManifestValidator.cs b/.github/workflows/unskip-closed-tests-tool/ManifestValidator.cs new file mode 100644 index 0000000000..60256beacc --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/ManifestValidator.cs @@ -0,0 +1,77 @@ +ο»Ώnamespace UnskipClosedTests.Tool; + +internal static class ManifestValidator +{ + public static Manifest Read(string path) + { + Manifest manifest = JsonSupport.Read(path); + Validate(manifest); + return manifest; + } + + public static void Validate(Manifest manifest) + { + if (manifest.SchemaVersion != "1") + { + throw new ContractException($"Unsupported manifest schema_version '{manifest.SchemaVersion}'."); + } + + if (manifest.CandidateCount != manifest.Candidates.Count) + { + throw new ContractException("candidate_count does not match candidates."); + } + + if (manifest.Repository.Split('/').Length != 2 || + manifest.SourceCommit.Length is not (40 or 64) || + manifest.GitObjectFormat is not ("sha1" or "sha256") || + manifest.ConfigDigest.Length != 64 || + manifest.ManifestDigest.Length != 64) + { + throw new ContractException("Manifest root identity fields are malformed."); + } + + if (!string.Equals(JsonSupport.ManifestDigest(manifest), manifest.ManifestDigest, StringComparison.Ordinal)) + { + throw new ContractException("manifest_digest does not match manifest content."); + } + + HashSet candidateIds = new(StringComparer.Ordinal); + foreach (Candidate candidate in manifest.Candidates) + { + if (!candidateIds.Add(candidate.CandidateId)) + { + throw new ContractException($"Duplicate candidate_id '{candidate.CandidateId}'."); + } + + PathRules.ValidateRelativePath(candidate.Path, "candidate path"); + if (!candidate.Path.EndsWith(".cs", StringComparison.OrdinalIgnoreCase) || + candidate.CandidateId.Length != 64 || + candidate.StableOwnerId.Length != 64 || + candidate.SourceSha256.Length != 64 || + candidate.AttributeTextSha256.Length != 64 || + candidate.AttributeSpan.Start < 0 || + candidate.AttributeSpan.Length <= 0 || + candidate.Owner.ContainingTypes.Count == 0 || + candidate.Owner.TypeFqn.Length == 0 || + candidate.Owner.DeclarationId.Length == 0) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has malformed trusted identity fields."); + } + + if (candidate.Owner.Kind is not ("method" or "class")) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has unsupported owner kind."); + } + + if (candidate.CanonicalIssueReferences.Count == 0) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has no concrete issue references."); + } + + if (candidate.Owner.TestFqns.Distinct(StringComparer.Ordinal).Count() != candidate.Owner.TestFqns.Count) + { + throw new ContractException($"Candidate '{candidate.CandidateId}' has duplicate test FQNs."); + } + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Models.cs b/.github/workflows/unskip-closed-tests-tool/Models.cs new file mode 100644 index 0000000000..0c05d6af17 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Models.cs @@ -0,0 +1,179 @@ +ο»Ώusing System.Text.Json.Serialization; + +namespace UnskipClosedTests.Tool; + +internal sealed class ToolConfig +{ + public string SchemaVersion { get; set; } = ""; + public List SourceRoots { get; set; } = []; + public List ExcludedGlobs { get; set; } = []; + public List GeneratedGlobs { get; set; } = []; + public List IgnoreAttributeNames { get; set; } = []; + public List TestAttributeNames { get; set; } = []; + public List VerificationCommand { get; set; } = []; + public int VerificationTimeoutSeconds { get; set; } +} + +internal sealed class Manifest +{ + public string SchemaVersion { get; set; } = "1"; + public string Repository { get; set; } = ""; + public string SourceCommit { get; set; } = ""; + public string GitObjectFormat { get; set; } = ""; + public string ConfigDigest { get; set; } = ""; + public string ManifestDigest { get; set; } = ""; + public int CandidateCount { get; set; } + public List Candidates { get; set; } = []; +} + +internal sealed class Candidate +{ + public string CandidateId { get; set; } = ""; + public string StableOwnerId { get; set; } = ""; + public string Path { get; set; } = ""; + public string BlobOid { get; set; } = ""; + public string SourceSha256 { get; set; } = ""; + public SourceSpan AttributeSpan { get; set; } = new(); + public string AttributeTextSha256 { get; set; } = ""; + public OwnerIdentity Owner { get; set; } = new(); + public List CanonicalIssueReferences { get; set; } = []; + public CandidateDecision Decision { get; set; } = new(); +} + +internal sealed class SourceSpan +{ + public int Start { get; set; } + public int Length { get; set; } + public int StartLine { get; set; } + public int StartColumn { get; set; } + public int EndLine { get; set; } + public int EndColumn { get; set; } +} + +internal sealed class OwnerIdentity +{ + public string Kind { get; set; } = ""; + public string Namespace { get; set; } = ""; + public List ContainingTypes { get; set; } = []; + public string TypeFqn { get; set; } = ""; + public string DeclarationId { get; set; } = ""; + public string MethodName { get; set; } = ""; + public string MethodSignature { get; set; } = ""; + public List TestFqns { get; set; } = []; +} + +internal sealed class IssueReference +{ + public string Kind { get; set; } = ""; + public string Owner { get; set; } = ""; + public string Repo { get; set; } = ""; + public int Number { get; set; } + public string Canonical { get; set; } = ""; + public string Url { get; set; } = ""; + public bool Eligibility { get; set; } + public string State { get; set; } = ""; + public string StateReason { get; set; } = ""; + public string? MergedAt { get; set; } +} + +internal sealed class CandidateDecision +{ + public bool Eligible { get; set; } + public List Deferrals { get; set; } = []; +} + +internal sealed class EvidenceFile +{ + public string SchemaVersion { get; set; } = ""; + public List References { get; set; } = []; +} + +internal sealed class EvidenceReference +{ + public string Canonical { get; set; } = ""; + public string Kind { get; set; } = ""; + public bool Accessible { get; set; } = true; + public string State { get; set; } = ""; + public string StateReason { get; set; } = ""; + public string? MergedAt { get; set; } +} + +internal sealed class ApplyRequest +{ + public string SchemaVersion { get; set; } = "1"; + public VerificationCandidateRequest Candidate { get; set; } = new(); + public string Repository { get; set; } = ""; + public string SourceCommit { get; set; } = ""; + public List Tests { get; set; } = []; +} + +internal sealed class VerificationCandidateRequest +{ + public string CandidateId { get; set; } = ""; +} + +internal sealed class VerificationTest +{ + public string Fqn { get; set; } = ""; + public string SourcePath { get; set; } = ""; + public string ResultFile { get; set; } = ""; +} + +internal sealed class ApplyResult +{ + public string SchemaVersion { get; set; } = "1"; + public string SourceCommit { get; set; } = ""; + public string ManifestDigest { get; set; } = ""; + public List RetainedCandidates { get; set; } = []; + public List RevertedCandidates { get; set; } = []; + public List ChangedFiles { get; set; } = []; + public List ChangedPaths { get; set; } = []; + public bool HasChanges { get; set; } + public string PrTitle { get; set; } = ""; + public string PrBody { get; set; } = ""; +} + +internal sealed class ChangedFileResult +{ + public string Path { get; set; } = ""; + public string ContentSha256 { get; set; } = ""; +} + +internal sealed class RetainedCandidateResult +{ + public string CandidateId { get; set; } = ""; + public string Path { get; set; } = ""; + public List TestFqns { get; set; } = []; +} + +internal sealed class RevertedCandidateResult +{ + public string CandidateId { get; set; } = ""; + public string Path { get; set; } = ""; + public List TestFqns { get; set; } = []; + public string Reason { get; set; } = ""; +} + +internal sealed class CliOptions +{ + public string Command { get; init; } = ""; + public Dictionary Values { get; init; } = new(StringComparer.Ordinal); + + public string Required(string name) => + Values.TryGetValue(name, out string? value) && value.Length > 0 + ? value + : throw new ContractException($"Missing required option --{name}."); + + public string? Optional(string name) => Values.GetValueOrDefault(name); +} + +internal sealed class ContractException(string message) : Exception(message); +internal sealed class InfrastructureException(string message, Exception? inner = null) : Exception(message, inner); + +internal static class ExitCodes +{ + public const int Success = 0; + public const int CleanNoOp = 10; + public const int Invalid = 20; + public const int Infrastructure = 30; +} diff --git a/.github/workflows/unskip-closed-tests-tool/PathRules.cs b/.github/workflows/unskip-closed-tests-tool/PathRules.cs new file mode 100644 index 0000000000..2edaadd0ad --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/PathRules.cs @@ -0,0 +1,72 @@ +ο»Ώusing System.Text.RegularExpressions; + +namespace UnskipClosedTests.Tool; + +internal static class PathRules +{ + public static string ValidateRelativePath(string value, string context) + { + if (string.IsNullOrWhiteSpace(value) || Path.IsPathRooted(value)) + { + throw new ContractException($"{context} '{value}' must be a non-empty repository-relative path."); + } + + string normalized = value.Replace('\\', '/'); + string[] parts = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0 || parts.Any(static part => part is "." or "..")) + { + throw new ContractException($"{context} '{value}' contains traversal or an empty path."); + } + + return string.Join('/', parts); + } + + public static string ResolveInsideRoot(string repoRoot, string relativePath, string context) + { + string normalized = ValidateRelativePath(relativePath, context); + string root = Path.GetFullPath(repoRoot); + string fullPath = Path.GetFullPath(Path.Combine(root, normalized.Replace('/', Path.DirectorySeparatorChar))); + string prefix = root.EndsWith(Path.DirectorySeparatorChar) ? root : root + Path.DirectorySeparatorChar; + if (!fullPath.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException($"{context} '{relativePath}' escapes the repository root."); + } + + return fullPath; + } + + public static void RejectReparsePoints(string repoRoot, string fullPath) + { + string root = Path.GetFullPath(repoRoot).TrimEnd(Path.DirectorySeparatorChar); + string current = Path.GetFullPath(fullPath); + while (!string.Equals(current, root, StringComparison.OrdinalIgnoreCase)) + { + if (!File.Exists(current) && !Directory.Exists(current)) + { + throw new ContractException($"Source path '{fullPath}' does not exist."); + } + + if ((File.GetAttributes(current) & FileAttributes.ReparsePoint) != 0) + { + throw new ContractException($"Source path '{fullPath}' traverses a symlink or reparse point."); + } + + current = Path.GetDirectoryName(current) + ?? throw new ContractException($"Source path '{fullPath}' is outside the repository."); + } + } + + public static bool MatchesAnyGlob(string path, IEnumerable globs) => + globs.Any(glob => GlobToRegex(glob).IsMatch(path)); + + private static Regex GlobToRegex(string glob) + { + string normalized = glob.Replace('\\', '/'); + string pattern = Regex.Escape(normalized) + .Replace(@"\*\*/", "(?:.*/)?", StringComparison.Ordinal) + .Replace(@"\*\*", ".*", StringComparison.Ordinal) + .Replace(@"\*", "[^/]*", StringComparison.Ordinal) + .Replace(@"\?", "[^/]", StringComparison.Ordinal); + return new Regex($"^{pattern}$", RegexOptions.CultureInvariant | RegexOptions.NonBacktracking); + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Program.cs b/.github/workflows/unskip-closed-tests-tool/Program.cs new file mode 100644 index 0000000000..6ca86ecbf2 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Program.cs @@ -0,0 +1,149 @@ +ο»Ώnamespace UnskipClosedTests.Tool; + +internal static class Program +{ + private const string HelpText = + """ + Usage: + UnskipClosedTests.Tool inventory --config --output [--repo-root ] [--repository ] [--source-commit ] + UnskipClosedTests.Tool resolve --manifest --output [--github-evidence ] + UnskipClosedTests.Tool apply --config --manifest --agent-output --output [--repo-root ] [--github-evidence ] + + Exit codes: + 0 Successful inventory/resolve, or apply retained at least one verified edit. + 10 Apply retained no verified candidates and left candidate source files unchanged. + 20 Invalid or stale trusted input. + 30 Infrastructure or verification protocol failure. + """; + + public static async Task Main(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine(HelpText); + return ExitCodes.Invalid; + } + + if (args[0] is "--help" or "-h" or "help" || + args.Length == 2 && args[1] is "--help" or "-h") + { + Console.WriteLine(HelpText); + return ExitCodes.Success; + } + + try + { + CliOptions options = Parse(args); + return options.Command switch + { + "inventory" => RunInventory(options), + "resolve" => await RunResolveAsync(options), + "apply" => await RunApplyAsync(options), + _ => throw new ContractException($"Unknown command '{options.Command}'. Expected inventory, resolve, or apply."), + }; + } + catch (ContractException ex) + { + Console.Error.WriteLine($"invalid: {ex.Message}"); + return ExitCodes.Invalid; + } + catch (InfrastructureException ex) + { + Console.Error.WriteLine($"infrastructure: {ex.Message}"); + return ExitCodes.Infrastructure; + } + catch (Exception ex) + { + Console.Error.WriteLine($"infrastructure: unexpected failure: {ex}"); + return ExitCodes.Infrastructure; + } + } + + private static int RunInventory(CliOptions options) + { + string configPath = options.Required("config"); + string outputPath = options.Required("output"); + string repoRoot = options.Optional("repo-root") ?? Environment.CurrentDirectory; + ToolConfig config = ConfigLoader.Load(configPath); + Manifest manifest = InventoryEngine.Create(repoRoot, options.Optional("repository"), config); + string? expectedSourceCommit = options.Optional("source-commit"); + if (expectedSourceCommit is not null && + !string.Equals(expectedSourceCommit, manifest.SourceCommit, StringComparison.OrdinalIgnoreCase)) + { + throw new ContractException( + $"Expected source commit '{expectedSourceCommit}' does not match checked-out commit '{manifest.SourceCommit}'."); + } + + JsonSupport.Write(outputPath, manifest); + return ExitCodes.Success; + } + + private static async Task RunResolveAsync(CliOptions options) + { + Manifest manifest = ManifestValidator.Read(options.Required("manifest")); + string outputPath = options.Required("output"); + Manifest resolved = await IssueResolver.ResolveAsync(manifest, options.Optional("github-evidence")); + JsonSupport.Write(outputPath, resolved); + return ExitCodes.Success; + } + + private static async Task RunApplyAsync(CliOptions options) + { + string configPath = options.Required("config"); + ToolConfig config = ConfigLoader.Load(configPath); + ApplyResult result = await ApplyEngine.ApplyAsync( + options.Optional("repo-root") ?? Environment.CurrentDirectory, + config, + options.Required("manifest"), + options.Required("agent-output"), + options.Optional("github-evidence")); + JsonSupport.Write(options.Required("output"), result); + return result.HasChanges ? ExitCodes.Success : ExitCodes.CleanNoOp; + } + + private static CliOptions Parse(string[] args) + { + if (args.Length == 0) + { + throw new ContractException("A command is required."); + } + + Dictionary values = new(StringComparer.Ordinal); + for (int i = 1; i < args.Length; i++) + { + string item = args[i]; + if (!item.StartsWith("--", StringComparison.Ordinal) || item.Length == 2) + { + throw new ContractException($"Unexpected argument '{item}'."); + } + + string name = item[2..]; + if (i + 1 >= args.Length || args[i + 1].StartsWith("--", StringComparison.Ordinal)) + { + throw new ContractException($"Option --{name} requires a value."); + } + + if (!values.TryAdd(name, args[++i])) + { + throw new ContractException($"Option --{name} was specified more than once."); + } + } + + HashSet allowed = args[0] switch + { + "inventory" => ["config", "output", "repo-root", "repository", "source-commit"], + "resolve" => ["manifest", "output", "github-evidence"], + "apply" => ["config", "manifest", "agent-output", "output", "repo-root", "github-evidence"], + _ => [], + }; + foreach (string name in values.Keys) + { + if (!allowed.Contains(name)) + { + throw new ContractException($"Option --{name} is not valid for command '{args[0]}'."); + } + } + + return new CliOptions { Command = args[0], Values = values }; + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Tests/Program.cs b/.github/workflows/unskip-closed-tests-tool/Tests/Program.cs new file mode 100644 index 0000000000..9f6f95fc52 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Tests/Program.cs @@ -0,0 +1,217 @@ +ο»Ώusing System.Diagnostics; +using System.Text; + +namespace UnskipClosedTests.Tool.Tests; + +internal static class Program +{ + private static readonly string[] RestrictedEnvironmentVariables = + [ + "ACTIONS_ID_TOKEN_REQUEST_TOKEN", + "ACTIONS_RUNTIME_TOKEN", + "GH_ENTERPRISE_TOKEN", + "GH_AW_AGENT_OUTPUT", + "GH_TOKEN", + "GITHUB_ENTERPRISE_TOKEN", + "GITHUB_ENV", + "GITHUB_OUTPUT", + "GITHUB_TOKEN", + "ORIGINAL_MANIFEST", + "RESULT_PATH", + ]; + + public static int Main() + { + List<(string Name, Action Test)> tests = + [ + ("rejects malformed issue reference suffixes", RejectsMalformedIssueReferenceSuffixes), + ("removes GitHub credentials from verification", RemovesGitHubCredentialsFromVerification), + ("records exact final content hashes", RecordsExactFinalContentHashes), + ("keeps duplicate detection marker in titles", KeepsDuplicateDetectionMarkerInTitles), + ("verifies every target-specific TRX", VerifiesEveryTargetSpecificTrx), + ]; + + foreach ((string name, Action test) in tests) + { + test(); + Console.WriteLine($"PASS: {name}"); + } + + Console.WriteLine($"All {tests.Count} unskip closed tests tool tests passed."); + return 0; + } + + private static void RejectsMalformedIssueReferenceSuffixes() + { + string repository = "microsoft/testfx"; + string[] malformed = + [ + "https://github.com/microsoft/testfx/issues/123abc", + "microsoft/testfx#456suffix", + "#789_identifier", + ]; + foreach (string value in malformed) + { + AssertEqual(0, InventoryEngine.ParseReferences(value, repository).Count(), $"Reference '{value}' was accepted."); + } + + List valid = InventoryEngine.ParseReferences( + "https://github.com/microsoft/testfx/issues/123, microsoft/testfx#456; #789.", + repository).ToList(); + AssertEqual(3, valid.Count, "Valid issue references were not all accepted."); + AssertEqual("microsoft/testfx#123", valid[0].Canonical, "Full issue URL was not canonicalized."); + AssertEqual("microsoft/testfx#456", valid[1].Canonical, "Qualified issue reference was not canonicalized."); + AssertEqual("microsoft/testfx#789", valid[2].Canonical, "Bare issue reference was not canonicalized."); + } + + private static void RemovesGitHubCredentialsFromVerification() + { + Dictionary originalValues = RestrictedEnvironmentVariables + .ToDictionary(static variable => variable, Environment.GetEnvironmentVariable, StringComparer.Ordinal); + const string sentinel = "UNSKIP_VERIFICATION_ENVIRONMENT_SENTINEL"; + string? originalSentinel = Environment.GetEnvironmentVariable(sentinel); + + try + { + foreach (string variable in RestrictedEnvironmentVariables) + { + Environment.SetEnvironmentVariable(variable, "secret"); + } + + Environment.SetEnvironmentVariable(sentinel, "retained"); + ProcessStartInfo startInfo = ApplyEngine.CreateVerificationStartInfo( + "dotnet", + Environment.CurrentDirectory, + ["--info"]); + + foreach (string variable in RestrictedEnvironmentVariables) + { + AssertFalse(startInfo.Environment.ContainsKey(variable), $"{variable} remained in the child environment."); + } + + AssertEqual("retained", startInfo.Environment[sentinel], "Non-sensitive environment was unexpectedly removed."); + AssertEqual("--info", startInfo.ArgumentList.Single(), "Verification argument was not preserved."); + } + finally + { + foreach ((string variable, string? value) in originalValues) + { + Environment.SetEnvironmentVariable(variable, value); + } + + Environment.SetEnvironmentVariable(sentinel, originalSentinel); + } + } + + private static void RecordsExactFinalContentHashes() + { + string temporaryRoot = Path.Combine(Path.GetTempPath(), $"unskip-tool-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temporaryRoot); + try + { + byte[] firstContent = [0xEF, 0xBB, 0xBF, .. Encoding.UTF8.GetBytes("first")]; + byte[] secondContent = Encoding.UTF8.GetBytes("second"); + File.WriteAllBytes(Path.Combine(temporaryRoot, "First.cs"), firstContent); + File.WriteAllBytes(Path.Combine(temporaryRoot, "Second.cs"), secondContent); + + List files = ApplyEngine.CreateChangedFiles( + temporaryRoot, + ["Second.cs", "First.cs", "Second.cs"]); + + AssertEqual(2, files.Count, "Changed files were not deduplicated."); + AssertEqual("First.cs", files[0].Path, "Changed files were not sorted."); + AssertEqual(JsonSupport.Sha256(firstContent), files[0].ContentSha256, "BOM-preserving hash was incorrect."); + AssertEqual("Second.cs", files[1].Path, "Second changed file was missing."); + AssertEqual(JsonSupport.Sha256(secondContent), files[1].ContentSha256, "Content hash was incorrect."); + } + finally + { + Directory.Delete(temporaryRoot, recursive: true); + } + } + + private static void KeepsDuplicateDetectionMarkerInTitles() + { + AssertTrue( + ApplyEngine.CreatePrTitle(1).StartsWith("[unskip-closed-tests] ", StringComparison.Ordinal), + "Single-candidate title is missing the duplicate-detection marker."); + AssertTrue( + ApplyEngine.CreatePrTitle(2).StartsWith("[unskip-closed-tests] ", StringComparison.Ordinal), + "Multi-candidate title is missing the duplicate-detection marker."); + } + + private static void VerifiesEveryTargetSpecificTrx() + { + string temporaryRoot = Path.Combine(Path.GetTempPath(), $"unskip-trx-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temporaryRoot); + try + { + string requestedResult = Path.Combine(temporaryRoot, "0000.trx"); + string firstResult = Path.Combine(temporaryRoot, "0000--net8.0.trx"); + string secondResult = Path.Combine(temporaryRoot, "0000--net9.0.trx"); + WriteTrx(firstResult, "first", "Passed"); + WriteTrx(secondResult, "second", "Passed"); + + VerificationTest test = new() + { + Fqn = "Example.Tests.TestOne", + ResultFile = requestedResult, + SourcePath = "Tests.cs", + }; + (bool success, string reason) = TrxVerifier.Verify([test]); + AssertTrue(success, $"Multi-target TRX verification failed: {reason}"); + + WriteTrx(secondResult, "second", "Failed"); + (success, reason) = TrxVerifier.Verify([test]); + AssertFalse(success, "A failing target-specific TRX was accepted."); + AssertTrue( + reason.StartsWith("non_passing_outcome:", StringComparison.Ordinal), + $"Unexpected failure reason: {reason}"); + } + finally + { + Directory.Delete(temporaryRoot, recursive: true); + } + } + + private static void WriteTrx(string path, string id, string outcome) => + File.WriteAllText( + path, + $""" + + + + + + + + + + + """, + new UTF8Encoding(false)); + + private static void AssertEqual(T expected, T actual, string message) + { + if (!EqualityComparer.Default.Equals(expected, actual)) + { + throw new InvalidOperationException($"{message} Expected '{expected}', actual '{actual}'."); + } + } + + private static void AssertFalse(bool condition, string message) + { + if (condition) + { + throw new InvalidOperationException(message); + } + } + + private static void AssertTrue(bool condition, string message) + { + if (!condition) + { + throw new InvalidOperationException(message); + } + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/Tests/UnskipClosedTests.Tool.Tests.csproj b/.github/workflows/unskip-closed-tests-tool/Tests/UnskipClosedTests.Tool.Tests.csproj new file mode 100644 index 0000000000..7f2cd67599 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/Tests/UnskipClosedTests.Tool.Tests.csproj @@ -0,0 +1,16 @@ + + + Exe + net8.0 + enable + enable + true + false + false + false + + + + + + diff --git a/.github/workflows/unskip-closed-tests-tool/TrxVerifier.cs b/.github/workflows/unskip-closed-tests-tool/TrxVerifier.cs new file mode 100644 index 0000000000..0b471c9bca --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/TrxVerifier.cs @@ -0,0 +1,140 @@ +ο»Ώusing System.Xml; +using System.Xml.Linq; + +namespace UnskipClosedTests.Tool; + +internal static class TrxVerifier +{ + public static (bool Success, string Reason) Verify(IReadOnlyList tests) + { + Dictionary expectedFiles = new(StringComparer.OrdinalIgnoreCase); + foreach (VerificationTest test in tests) + { + string requestedFile = Path.GetFullPath(test.ResultFile); + string directory = Path.GetDirectoryName(requestedFile)!; + string requestedName = Path.GetFileName(requestedFile); + string targetPrefix = $"{Path.GetFileNameWithoutExtension(requestedFile)}--"; + List resultFiles = Directory.Exists(directory) + ? Directory.EnumerateFiles(directory, "*.trx", SearchOption.TopDirectoryOnly) + .Select(Path.GetFullPath) + .Where(path => + string.Equals(Path.GetFileName(path), requestedName, StringComparison.OrdinalIgnoreCase) || + Path.GetFileName(path).StartsWith(targetPrefix, StringComparison.OrdinalIgnoreCase)) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList() + : []; + if (resultFiles.Count == 0) + { + return (false, $"missing_trx:{requestedName}"); + } + + foreach (string resultFile in resultFiles) + { + if (!expectedFiles.TryAdd(resultFile, test)) + { + return (false, $"duplicate_trx_path:{Path.GetFileName(resultFile)}"); + } + } + } + + foreach (string directory in tests + .Select(static test => Path.GetDirectoryName(Path.GetFullPath(test.ResultFile))!) + .Distinct(StringComparer.OrdinalIgnoreCase)) + { + if (Directory.Exists(directory)) + { + string? unexpected = Directory.EnumerateFiles(directory, "*.trx", SearchOption.TopDirectoryOnly) + .Select(Path.GetFullPath) + .FirstOrDefault(path => !expectedFiles.ContainsKey(path)); + if (unexpected is not null) + { + return (false, $"unexpected_trx:{Path.GetFileName(unexpected)}"); + } + } + } + + foreach ((string resultFile, VerificationTest test) in expectedFiles) + { + try + { + XDocument document = XDocument.Load(resultFile, LoadOptions.None); + Dictionary mappings = new(StringComparer.Ordinal); + foreach (XElement unitTest in document.Descendants().Where(static element => + element.Name.LocalName == "UnitTest")) + { + string? id = unitTest.Attribute("id")?.Value; + XElement? method = unitTest.Descendants().FirstOrDefault(static element => + element.Name.LocalName == "TestMethod"); + string? className = method?.Attribute("className")?.Value; + string? methodName = method?.Attribute("name")?.Value; + if (string.IsNullOrWhiteSpace(id) || + string.IsNullOrWhiteSpace(className) || + string.IsNullOrWhiteSpace(methodName)) + { + return (false, "malformed_trx_test_definition"); + } + + string fqn = $"{className}.{methodName}"; + if (!string.Equals(fqn, test.Fqn, StringComparison.Ordinal)) + { + return (false, $"mismatched_fqn:{fqn}"); + } + + if (!mappings.TryAdd(id, fqn)) + { + return (false, $"duplicate_trx_test_id:{id}"); + } + } + + if (mappings.Count == 0) + { + return (false, "zero_selected_tests"); + } + + List results = document.Descendants().Where(static element => + element.Name.LocalName == "UnitTestResult").ToList(); + if (results.Count == 0) + { + return (false, "zero_executed_tests"); + } + + int passed = 0; + foreach (XElement result in results) + { + string? testId = result.Attribute("testId")?.Value; + string? outcome = result.Attribute("outcome")?.Value; + if (string.IsNullOrWhiteSpace(testId) || + !mappings.TryGetValue(testId, out string? mappedFqn)) + { + return (false, "result_without_exact_test_mapping"); + } + + if (!string.Equals(mappedFqn, test.Fqn, StringComparison.Ordinal)) + { + return (false, $"mismatched_result_fqn:{mappedFqn}"); + } + + if (string.Equals(outcome, "Passed", StringComparison.OrdinalIgnoreCase)) + { + passed++; + } + else + { + return (false, $"non_passing_outcome:{outcome ?? "missing"}"); + } + } + + if (passed == 0) + { + return (false, "no_executed_pass"); + } + } + catch (Exception ex) when (ex is XmlException or IOException or UnauthorizedAccessException) + { + return (false, $"malformed_trx:{ex.GetType().Name}"); + } + } + + return (true, "passed"); + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj b/.github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj new file mode 100644 index 0000000000..ea4f5c4a15 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/UnskipClosedTests.Tool.csproj @@ -0,0 +1,20 @@ +ο»Ώ + + Exe + net8.0 + enable + enable + true + true + true + + false + false + false + + + + + + + diff --git a/.github/workflows/unskip-closed-tests-tool/global.json b/.github/workflows/unskip-closed-tests-tool/global.json new file mode 100644 index 0000000000..4c4c3ae5ed --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.100", + "rollForward": "latestFeature", + "allowPrerelease": false + } +} diff --git a/.github/workflows/unskip-closed-tests-tool/packages.lock.json b/.github/workflows/unskip-closed-tests-tool/packages.lock.json new file mode 100644 index 0000000000..0c4bdcbb56 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-tool/packages.lock.json @@ -0,0 +1,47 @@ +{ + "version": 1, + "dependencies": { + "net8.0": { + "Microsoft.CodeAnalysis.CSharp": { + "type": "Direct", + "requested": "[4.14.0, )", + "resolved": "4.14.0", + "contentHash": "568a6wcTivauIhbeWcCwfWwIn7UV7MeHEBvFB2uzGIpM2OhJ4eM/FZ8KS0yhPoNxnSpjGzz7x7CIjTxhslojQA==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "Microsoft.CodeAnalysis.Common": "[4.14.0]", + "System.Collections.Immutable": "9.0.0", + "System.Reflection.Metadata": "9.0.0" + } + }, + "Microsoft.CodeAnalysis.Analyzers": { + "type": "Transitive", + "resolved": "3.11.0", + "contentHash": "v/EW3UE8/lbEYHoC2Qq7AR/DnmvpgdtAMndfQNmpuIMx/Mto8L5JnuCfdBYtgvalQOtfNCnxFejxuRrryvUTsg==" + }, + "Microsoft.CodeAnalysis.Common": { + "type": "Transitive", + "resolved": "4.14.0", + "contentHash": "PC3tuwZYnC+idaPuoC/AZpEdwrtX7qFpmnrfQkgobGIWiYmGi5MCRtl5mx6QrfMGQpK78X2lfIEoZDLg/qnuHg==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "System.Collections.Immutable": "9.0.0", + "System.Reflection.Metadata": "9.0.0" + } + }, + "System.Collections.Immutable": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "QhkXUl2gNrQtvPmtBTQHb0YsUrDiDQ2QS09YbtTTiSjGcf7NBqtYbrG/BE06zcBPCKEwQGzIv13IVdXNOSub2w==" + }, + "System.Reflection.Metadata": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "ANiqLu3DxW9kol/hMmTWbt3414t9ftdIuiIU7j80okq2YzAueo120M442xk1kDJWtmZTqWQn7wHDvMRipVOEOQ==", + "dependencies": { + "System.Collections.Immutable": "9.0.0" + } + } + } + } +} \ No newline at end of file diff --git a/.github/workflows/unskip-closed-tests-verify.ps1 b/.github/workflows/unskip-closed-tests-verify.ps1 new file mode 100644 index 0000000000..991e52dcd9 --- /dev/null +++ b/.github/workflows/unskip-closed-tests-verify.ps1 @@ -0,0 +1,651 @@ +param( + [Parameter(Position = 0, Mandatory)] + [string] $RequestPath, + + [string] $RepositoryRoot = (Get-Location).Path, + + [ValidateRange(1, 86400)] + [int] $TimeoutSeconds = 1800 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$script:MaxRequestBytes = 4 * 1024 * 1024 +$script:FqnPattern = '^(?:@?[A-Za-z_][A-Za-z0-9_]*\.)+@?[A-Za-z_][A-Za-z0-9_]*$' +$script:TfmPattern = '^net(?[0-9]+)(?:\.[0-9]+)?(?:-[A-Za-z0-9.-]+)?$' + +function Get-RequiredProperty { + param( + [Parameter(Mandatory)] [object] $InputObject, + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [string] $Context + ) + + $property = $InputObject.PSObject.Properties[$Name] + if ($null -eq $property) { + throw "$Context.$Name is required." + } + + return $property.Value +} + +function Get-RequiredString { + param( + [Parameter(Mandatory)] [object] $InputObject, + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [string] $Context + ) + + $value = Get-RequiredProperty -InputObject $InputObject -Name $Name -Context $Context + if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace($value)) { + throw "$Context.$Name must be a non-empty string." + } + + return $value +} + +function Get-RequiredArray { + param( + [Parameter(Mandatory)] [object] $InputObject, + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [string] $Context + ) + + $value = Get-RequiredProperty -InputObject $InputObject -Name $Name -Context $Context + if ($value -is [string]) { + throw "$Context.$Name must be an array." + } + + return @($value) +} + +function ConvertTo-RepositoryRelativePath { + param( + [Parameter(Mandatory)] [string] $Value, + [Parameter(Mandatory)] [string] $Context + ) + + if ( + [System.IO.Path]::IsPathRooted($Value) -or + $Value.Contains('\') -or + $Value.StartsWith('./', [StringComparison]::Ordinal) -or + ($Value -split '/') -contains '..' + ) { + throw "$Context must be a normalized repository-relative path using '/' separators." + } + + return $Value +} + +function Read-VerificationRequest { + param([Parameter(Mandatory)] [string] $Path) + + $item = Get-Item -LiteralPath $Path -ErrorAction Stop + if ($item.Length -gt $script:MaxRequestBytes) { + throw "Verification request exceeds the $($script:MaxRequestBytes)-byte limit." + } + + try { + $request = Get-Content -LiteralPath $item.FullName -Raw -Encoding utf8 | + ConvertFrom-Json -Depth 64 + } + catch { + throw "Cannot read verification request: $($_.Exception.Message)" + } + + if ((Get-RequiredString -InputObject $request -Name 'schema_version' -Context 'verification request') -ne '1') { + throw "verification request.schema_version must be '1'." + } + + $candidate = Get-RequiredProperty -InputObject $request -Name 'candidate' -Context 'verification request' + $candidateId = Get-RequiredString -InputObject $candidate -Name 'candidate_id' -Context 'verification request.candidate' + $repository = Get-RequiredString -InputObject $request -Name 'repository' -Context 'verification request' + if ($repository -notmatch '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$') { + throw 'verification request.repository must use owner/repository form.' + } + + $sourceCommit = ( + Get-RequiredString -InputObject $request -Name 'source_commit' -Context 'verification request' + ).ToLowerInvariant() + if ($sourceCommit -notmatch '^[0-9a-f]{40,64}$') { + throw 'verification request.source_commit must be a full hexadecimal object id.' + } + + $tests = @() + $seen = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $index = 0 + foreach ($test in Get-RequiredArray -InputObject $request -Name 'tests' -Context 'verification request') { + $context = "verification request.tests[$index]" + $fqn = Get-RequiredString -InputObject $test -Name 'fqn' -Context $context + if ($fqn -notmatch $script:FqnPattern) { + throw "$context.fqn is not a supported test identity." + } + if (-not $seen.Add($fqn)) { + throw "verification request contains duplicate test identity '$fqn'." + } + + $sourcePath = ConvertTo-RepositoryRelativePath -Value ( + Get-RequiredString -InputObject $test -Name 'source_path' -Context $context + ) -Context "$context.source_path" + if (-not $sourcePath.EndsWith('.cs', [StringComparison]::OrdinalIgnoreCase)) { + throw "$context.source_path must identify C# source." + } + + $resultFile = Get-RequiredString -InputObject $test -Name 'result_file' -Context $context + $tests += [pscustomobject]@{ + Fqn = $fqn + SourcePath = $sourcePath + ResultFile = $resultFile + } + $index++ + } + + if ($tests.Count -eq 0) { + throw 'verification request.tests must not be empty.' + } + + return [pscustomobject]@{ + CandidateId = $candidateId + Repository = $repository + SourceCommit = $sourceCommit + Tests = $tests + } +} + +function Invoke-Git { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string[]] $Arguments + ) + + $output = & git -C $Root @Arguments 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "git $($Arguments -join ' ') failed: $($output -join [Environment]::NewLine)" + } + + return ($output -join [Environment]::NewLine).Trim() +} + +function Assert-Revision { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string] $ExpectedCommit + ) + + $head = (Invoke-Git -Root $Root -Arguments @('rev-parse', 'HEAD')).ToLowerInvariant() + if ($head -ne $ExpectedCommit) { + throw "Repository revision changed from $ExpectedCommit to $head." + } +} + +function Get-TestProject { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string] $SourcePath + ) + + $resolvedRoot = [System.IO.Path]::GetFullPath($Root) + $relative = $SourcePath.Replace('/', [System.IO.Path]::DirectorySeparatorChar) + $source = [System.IO.Path]::GetFullPath((Join-Path $resolvedRoot $relative)) + $prefix = $resolvedRoot.TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, + [System.IO.Path]::AltDirectorySeparatorChar + ) + [System.IO.Path]::DirectorySeparatorChar + if (-not $source.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Source file escapes the repository: $SourcePath" + } + if (-not (Test-Path -LiteralPath $source -PathType Leaf)) { + throw "Source file does not exist: $SourcePath" + } + + $directory = Split-Path -Parent $source + while ($true) { + $projects = @(Get-ChildItem -LiteralPath $directory -Filter '*.csproj' -File) + if ($projects.Count -eq 1) { + return $projects[0].FullName + } + if ($projects.Count -gt 1) { + throw "Source project is ambiguous for ${SourcePath}: $($projects.Name -join ', ')" + } + if ([string]::Equals($directory, $resolvedRoot, [StringComparison]::OrdinalIgnoreCase)) { + break + } + $parent = Split-Path -Parent $directory + if ([string]::IsNullOrEmpty($parent) -or $parent -eq $directory) { + break + } + $directory = $parent + } + + throw "No owning test project found for $SourcePath." +} + +function Get-DotNetPath { + param([Parameter(Mandatory)] [string] $Root) + + $executable = if ($IsWindows) { 'dotnet.exe' } else { 'dotnet' } + $local = Join-Path (Join-Path $Root '.dotnet') $executable + if (Test-Path -LiteralPath $local -PathType Leaf) { + return $local + } + + return 'dotnet' +} + +function New-ProcessStartInfo { + param( + [Parameter(Mandatory)] [string] $FileName, + [Parameter(Mandatory)] [string[]] $Arguments, + [Parameter(Mandatory)] [string] $WorkingDirectory, + [switch] $CaptureOutput + ) + + $startInfo = [System.Diagnostics.ProcessStartInfo]::new() + $startInfo.FileName = $FileName + $startInfo.WorkingDirectory = $WorkingDirectory + $startInfo.UseShellExecute = $false + $startInfo.CreateNoWindow = $true + $startInfo.RedirectStandardOutput = $CaptureOutput + $startInfo.RedirectStandardError = $CaptureOutput + $startInfo.Environment['DOTNET_ROLL_FORWARD'] = if ($env:DOTNET_ROLL_FORWARD) { + $env:DOTNET_ROLL_FORWARD + } else { + 'Major' + } + $startInfo.Environment['DOTNET_ROLL_FORWARD_TO_PRERELEASE'] = if ($env:DOTNET_ROLL_FORWARD_TO_PRERELEASE) { + $env:DOTNET_ROLL_FORWARD_TO_PRERELEASE + } else { + '1' + } + foreach ($argument in $Arguments) { + [void] $startInfo.ArgumentList.Add($argument) + } + + return $startInfo +} + +function Invoke-CheckedProcess { + param( + [Parameter(Mandatory)] [string] $FileName, + [Parameter(Mandatory)] [string[]] $Arguments, + [Parameter(Mandatory)] [string] $WorkingDirectory, + [Parameter(Mandatory)] [int] $Timeout + ) + + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = New-ProcessStartInfo -FileName $FileName -Arguments $Arguments -WorkingDirectory $WorkingDirectory + try { + if (-not $process.Start()) { + throw "Could not start command '$FileName'." + } + if (-not $process.WaitForExit($Timeout * 1000)) { + $process.Kill($true) + $process.WaitForExit() + throw "Command timed out after $Timeout seconds: $FileName $($Arguments -join ' ')" + } + if ($process.ExitCode -ne 0) { + throw "Command exited with $($process.ExitCode): $FileName $($Arguments -join ' ')" + } + } + finally { + $process.Dispose() + } +} + +function Invoke-CapturedProcess { + param( + [Parameter(Mandatory)] [string] $FileName, + [Parameter(Mandatory)] [string[]] $Arguments, + [Parameter(Mandatory)] [string] $WorkingDirectory, + [Parameter(Mandatory)] [int] $Timeout + ) + + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = New-ProcessStartInfo -FileName $FileName -Arguments $Arguments -WorkingDirectory $WorkingDirectory -CaptureOutput + try { + if (-not $process.Start()) { + throw "Could not start command '$FileName'." + } + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit($Timeout * 1000)) { + $process.Kill($true) + $process.WaitForExit() + throw "Command timed out after $Timeout seconds: $FileName $($Arguments -join ' ')" + } + $output = $stdout.GetAwaiter().GetResult() + $errorOutput = $stderr.GetAwaiter().GetResult() + if ($process.ExitCode -ne 0) { + throw "Command exited with $($process.ExitCode): $FileName $($Arguments -join ' '): $errorOutput" + } + + return $output + } + finally { + $process.Dispose() + } +} + +function Select-TargetFrameworks { + param([Parameter(Mandatory)] [string[]] $Frameworks) + + $normalized = @($Frameworks | Where-Object { + -not [string]::IsNullOrWhiteSpace($_) + } | ForEach-Object { + $_.Trim() + } | Sort-Object -Unique) + if ($normalized.Count -eq 0) { + throw 'Test project does not declare any target frameworks.' + } + + $supported = @() + $unsupported = @() + foreach ($framework in $normalized) { + $match = [regex]::Match($framework, $script:TfmPattern) + if ($match.Success -and $framework.Contains('.') -and -not $framework.Contains('-')) { + $supported += [pscustomobject]@{ + Version = [int] $match.Groups['version'].Value + Framework = $framework + } + } else { + $unsupported += $framework + } + } + if ($unsupported.Count -gt 0) { + throw "Cannot verify every target framework on this runner: $($unsupported -join ', ')." + } + + return @($supported | Sort-Object Version, Framework | ForEach-Object { $_.Framework }) +} + +function Get-ProjectTargetFrameworks { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string] $Project, + [Parameter(Mandatory)] [int] $Timeout + ) + + $dotnet = Get-DotNetPath -Root $Root + $output = Invoke-CapturedProcess -FileName $dotnet -Arguments @( + 'msbuild', + $Project, + '-nologo', + '-getProperty:TargetFrameworks', + '-getProperty:TargetFramework', + '-getProperty:OutputType' + ) -WorkingDirectory $Root -Timeout $Timeout + + $jsonStart = $output.IndexOf('{') + if ($jsonStart -lt 0) { + throw 'Cannot parse evaluated test project properties.' + } + try { + $properties = ($output.Substring($jsonStart) | ConvertFrom-Json -Depth 16).Properties + } + catch { + throw "Cannot parse evaluated test project properties: $($_.Exception.Message)" + } + if (-not [string]::Equals([string] $properties.OutputType, 'Exe', [StringComparison]::OrdinalIgnoreCase)) { + throw "$Project is not an executable test project." + } + + $frameworkText = if ([string]::IsNullOrWhiteSpace([string] $properties.TargetFrameworks)) { + [string] $properties.TargetFramework + } else { + [string] $properties.TargetFrameworks + } + return @(Select-TargetFrameworks -Frameworks @($frameworkText -split ';' | Where-Object { $_ })) +} + +function Test-RequiresPackedPackages { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string] $Project + ) + + $relative = [System.IO.Path]::GetRelativePath($Root, $Project) + return @($relative -split '[\\/]' | Where-Object { + $_.EndsWith('.Acceptance.IntegrationTests', [StringComparison]::Ordinal) + }).Count -gt 0 +} + +function Initialize-RepositoryBuild { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string] $SourceCommit, + [Parameter(Mandatory)] [bool] $RequiresPack, + [Parameter(Mandatory)] [int] $Timeout + ) + + $temporaryRoot = if ($env:RUNNER_TEMP) { + $env:RUNNER_TEMP + } else { + Join-Path $Root 'artifacts/tmp' + } + $markerDirectory = Join-Path $temporaryRoot 'testfx-unskip' + $kind = if ($RequiresPack) { 'packed' } else { 'built' } + $marker = Join-Path $markerDirectory "$kind-$SourceCommit" + if (Test-Path -LiteralPath $marker -PathType Leaf) { + return + } + + if ($IsWindows) { + $buildScript = Join-Path $Root 'build.cmd' + $fileName = if ($env:COMSPEC) { $env:COMSPEC } else { 'cmd.exe' } + $arguments = @('/d', '/c', $buildScript) + } else { + $buildScript = Join-Path $Root 'build.sh' + $fileName = $buildScript + $arguments = @() + } + if (-not (Test-Path -LiteralPath $buildScript -PathType Leaf)) { + throw "Repository build script is missing: $buildScript" + } + if ($RequiresPack) { + $arguments += '-pack' + } + + Invoke-CheckedProcess -FileName $fileName -Arguments $arguments -WorkingDirectory $Root -Timeout $Timeout + Assert-Revision -Root $Root -ExpectedCommit $SourceCommit + [void] (New-Item -ItemType Directory -Path $markerDirectory -Force) + Set-Content -LiteralPath $marker -Value $SourceCommit -Encoding utf8NoBOM + if ($RequiresPack) { + Set-Content -LiteralPath (Join-Path $markerDirectory "built-$SourceCommit") -Value $SourceCommit -Encoding utf8NoBOM + } +} + +function Resolve-ResultPath { + param( + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [string] $Value + ) + + $path = if ([System.IO.Path]::IsPathRooted($Value)) { + [System.IO.Path]::GetFullPath($Value) + } else { + [System.IO.Path]::GetFullPath((Join-Path $Root $Value)) + } + $runnerOutputRoot = [System.IO.Path]::GetFullPath( + $(if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { $Root }) + ) + $isInsideRunnerOutput = [string]::Equals( + $path, + $runnerOutputRoot, + [StringComparison]::OrdinalIgnoreCase + ) -or $path.StartsWith( + $runnerOutputRoot.TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, + [System.IO.Path]::AltDirectorySeparatorChar + ) + [System.IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase + ) + if (-not $isInsideRunnerOutput) { + $gitDirectory = Invoke-Git -Root $Root -Arguments @('rev-parse', '--git-dir') + if (-not [System.IO.Path]::IsPathRooted($gitDirectory)) { + $gitDirectory = Join-Path $Root $gitDirectory + } + $metadataOutputRoot = [System.IO.Path]::GetFullPath( + (Join-Path $gitDirectory 'unskip-closed-tests') + ) + $isInsideMetadataOutput = [string]::Equals( + $path, + $metadataOutputRoot, + [StringComparison]::OrdinalIgnoreCase + ) -or $path.StartsWith( + $metadataOutputRoot.TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, + [System.IO.Path]::AltDirectorySeparatorChar + ) + [System.IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase + ) + if (-not $isInsideMetadataOutput) { + throw "Requested result file is outside the trusted output roots: $Value" + } + } + if (-not $path.EndsWith('.trx', [StringComparison]::OrdinalIgnoreCase)) { + throw "Requested result file must use .trx: $Value" + } + + return $path +} + +function Get-BuildArguments { + param( + [Parameter(Mandatory)] [string] $Project, + [Parameter(Mandatory)] [string] $TargetFramework + ) + + return @( + 'build', + $Project, + '-c', + 'Debug', + '-f', + $TargetFramework, + '--no-restore', + '-p:EnableCodeCoverage=False', + '-bl:{}' + ) +} + +function Get-TestArguments { + param( + [Parameter(Mandatory)] [string] $Project, + [Parameter(Mandatory)] [string] $TargetFramework, + [Parameter(Mandatory)] [string] $Fqn, + [Parameter(Mandatory)] [string] $ResultFile + ) + + return @( + 'run', + '--project', + $Project, + '-c', + 'Debug', + '-f', + $TargetFramework, + '--no-build', + '--no-restore', + '-p:EnableCodeCoverage=False', + '-bl:{}', + '--', + '--filter-uid', + $Fqn, + '--report-trx', + '--report-trx-filename', + [System.IO.Path]::GetFileName($ResultFile), + '--results-directory', + [System.IO.Path]::GetDirectoryName($ResultFile) + ) +} + +function Get-TargetResultFile { + param( + [Parameter(Mandatory)] [string] $RequestedResultFile, + [Parameter(Mandatory)] [string] $TargetFramework, + [Parameter(Mandatory)] [int] $TargetFrameworkCount + ) + + if ($TargetFrameworkCount -eq 1) { + return $RequestedResultFile + } + + $directory = [System.IO.Path]::GetDirectoryName($RequestedResultFile) + $stem = [System.IO.Path]::GetFileNameWithoutExtension($RequestedResultFile) + return Join-Path $directory "$stem--$TargetFramework.trx" +} + +function Invoke-UnskipVerification { + param( + [Parameter(Mandatory)] [string] $Path, + [Parameter(Mandatory)] [string] $Root, + [Parameter(Mandatory)] [int] $Timeout + ) + + $resolvedRoot = [System.IO.Path]::GetFullPath($Root) + $request = Read-VerificationRequest -Path $Path + Assert-Revision -Root $resolvedRoot -ExpectedCommit $request.SourceCommit + + $projects = @($request.Tests | ForEach-Object { + Get-TestProject -Root $resolvedRoot -SourcePath $_.SourcePath + } | Sort-Object -Unique) + if ($projects.Count -ne 1) { + throw 'A single verification request must map to exactly one test project.' + } + $project = $projects[0] + $requiresPack = Test-RequiresPackedPackages -Root $resolvedRoot -Project $project + Initialize-RepositoryBuild -Root $resolvedRoot -SourceCommit $request.SourceCommit -RequiresPack $requiresPack -Timeout $Timeout + $targetFrameworks = @( + Get-ProjectTargetFrameworks -Root $resolvedRoot -Project $project -Timeout $Timeout + ) + + $dotnet = Get-DotNetPath -Root $resolvedRoot + foreach ($targetFramework in $targetFrameworks) { + Invoke-CheckedProcess -FileName $dotnet -Arguments ( + Get-BuildArguments -Project $project -TargetFramework $targetFramework + ) -WorkingDirectory $resolvedRoot -Timeout $Timeout + Assert-Revision -Root $resolvedRoot -ExpectedCommit $request.SourceCommit + } + + $capturedResults = @{} + foreach ($test in $request.Tests) { + $requestedResultFile = Resolve-ResultPath -Root $resolvedRoot -Value $test.ResultFile + $resultDirectory = [System.IO.Path]::GetDirectoryName($requestedResultFile) + $resultStem = [System.IO.Path]::GetFileNameWithoutExtension($requestedResultFile) + [void] (New-Item -ItemType Directory -Path $resultDirectory -Force) + Remove-Item -LiteralPath $requestedResultFile -Force -ErrorAction SilentlyContinue + Get-ChildItem -LiteralPath $resultDirectory -Filter "$resultStem--*.trx" -File -ErrorAction SilentlyContinue | + Remove-Item -Force + + foreach ($targetFramework in $targetFrameworks) { + $resultFile = Get-TargetResultFile ` + -RequestedResultFile $requestedResultFile ` + -TargetFramework $targetFramework ` + -TargetFrameworkCount $targetFrameworks.Count + Invoke-CheckedProcess -FileName $dotnet -Arguments ( + Get-TestArguments -Project $project -TargetFramework $targetFramework -Fqn $test.Fqn -ResultFile $resultFile + ) -WorkingDirectory $resolvedRoot -Timeout $Timeout + if (-not (Test-Path -LiteralPath $resultFile -PathType Leaf)) { + throw "Test runner did not create requested TRX for ${targetFramework}: $resultFile" + } + Assert-Revision -Root $resolvedRoot -ExpectedCommit $request.SourceCommit + $capturedResults[$resultFile] = [System.IO.File]::ReadAllBytes($resultFile) + } + } + + foreach ($entry in $capturedResults.GetEnumerator()) { + [System.IO.File]::WriteAllBytes([string] $entry.Key, [byte[]] $entry.Value) + } +} + +if ($MyInvocation.InvocationName -ne '.') { + try { + Invoke-UnskipVerification -Path $RequestPath -Root $RepositoryRoot -Timeout $TimeoutSeconds + exit 0 + } + catch { + [Console]::Error.WriteLine("error: $($_.Exception.Message)") + exit 1 + } +} diff --git a/.github/workflows/unskip-closed-tests.config.json b/.github/workflows/unskip-closed-tests.config.json new file mode 100644 index 0000000000..364689ffa7 --- /dev/null +++ b/.github/workflows/unskip-closed-tests.config.json @@ -0,0 +1,42 @@ +{ + "schema_version": "1", + "source_roots": [ + "test", + "samples" + ], + "excluded_globs": [ + "**/bin/**", + "**/obj/**" + ], + "generated_globs": [ + "**/Generated/**", + "**/generated/**", + "**/*.Designer.cs", + "**/*.g.cs", + "**/*.generated.cs" + ], + "ignore_attribute_names": [ + "Ignore", + "IgnoreAttribute" + ], + "test_attribute_names": [ + "TestMethod", + "TestMethodAttribute", + "DataTestMethod", + "DataTestMethodAttribute", + "STATestMethod", + "STATestMethodAttribute", + "UITestMethod", + "UITestMethodAttribute" + ], + "verification": { + "command": [ + "pwsh", + "-NoLogo", + "-NoProfile", + "-File", + ".github/workflows/unskip-closed-tests-verify.ps1" + ], + "timeout_seconds": 1800 + } +} diff --git a/.github/workflows/unskip-closed-tests.lock.yml b/.github/workflows/unskip-closed-tests.lock.yml index 06669062e1..0e4967828f 100644 --- a/.github/workflows/unskip-closed-tests.lock.yml +++ b/.github/workflows/unskip-closed-tests.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a2670f95b69128450276ecb60b5278111ad9dcff9b0a63a51266121d511f1575","body_hash":"55b2626bdc15c282ff69e139f600d4d1d317542ca927a24385ae39bc6da98a31","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","detection_agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} -# gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.12.2","digest":"sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6","pinned_image":"ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_pull_request","missing_data","missing_tool","noop"]}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"00e472b5dd004b0c8979cd063264259625f227f3c49e3af62477c26baae570e7","body_hash":"1de573c9bf1109cfd88d9d55ea0b60931795655ec398306d4dc13841a2ccd09b","compiler_version":"v0.89.21","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.87"}} +# gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-dotnet","sha":"a98b56852c35b8e3190ac28c8c2271da59106c68","version":"v6.0.0 (source v6)"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"924af5fdc64061cfbf66fb584c8b07e2ac230c60","version":"v0.89.21"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23","digest":"sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23","digest":"sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23","digest":"sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.25","digest":"sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"}],"mcp_servers":[{"name":"safeoutputs","tools":["apply_verified_unskips","missing_data","missing_tool","noop"]}]} # This file was automatically generated by gh-aw (v0.89.21). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -17,20 +17,22 @@ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # # -# To update this file, edit the corresponding .md file and run: +# To update this file, edit dotnet/skills/agentic-workflows/unskip-closed-tests@57e48d3619bf44b9307a3197239d398b0287b25c and run: # gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Scans the test suite for skipped/ignored tests whose linked tracking issue is already closed, then opens a pull request that re-enables (unskips) them. +# Inventories source-bound .NET Ignore attributes at one trusted revision, permits a read-only agent to select only verified sites, and opens at most one draft pull request after deterministic issue and test-result validation. +# +# Source: dotnet/skills/agentic-workflows/unskip-closed-tests@57e48d3619bf44b9307a3197239d398b0287b25c # # Resolved workflow manifest: # Imports: -# - shared/repo-build-setup.md +# - unskip-closed-tests-prepare.md +# - unskip-closed-tests-shared.md # # Secrets used: -# - GH_AW_CI_TRIGGER_TOKEN # - GH_AW_DEFAULT_OTLP_ENDPOINT # - GH_AW_DEFAULT_OTLP_HEADERS # - GH_AW_GITHUB_MCP_SERVER_TOKEN @@ -39,11 +41,13 @@ # # Custom actions used: # - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 -# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 (source v6) # - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) # - github/gh-aw-actions/setup@924af5fdc64061cfbf66fb584c8b07e2ac230c60 # v0.89.21 # # Container images used: @@ -52,13 +56,12 @@ # - ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0 # - ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086 # - ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 -# - ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6 name: "Unskip Closed Tests" on: schedule: - cron: "50 20 * * 0" # Friendly format: weekly (scattered) - # skip-if-match: is:pr is:open in:title "[unskip-tests]" # Skip-if-match processed as search check in pre-activation job + # skip-if-match: is:pr is:open in:title "[unskip-closed-tests]" # Skip-if-match processed as search check in pre-activation job workflow_dispatch: inputs: aw_context: @@ -125,6 +128,7 @@ jobs: GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/unskip-closed-tests.lock.yml@${{ github.ref }} GH_AW_INFO_VERSION: "1.0.87" GH_AW_INFO_AWF_VERSION: "v0.28.23" + GH_AW_INFO_BODY_MODIFIED: "false" GH_AW_INFO_ENGINE_ID: "copilot" - name: Mask OTLP telemetry headers run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" @@ -147,6 +151,8 @@ jobs: GH_AW_INFO_AWMG_VERSION: "" GH_AW_INFO_FIREWALL_TYPE: "squid" GH_AW_INFO_AGENT_RUNTIME: "" + GH_AW_INFO_FRONTMATTER_SOURCE: "dotnet/skills/agentic-workflows/unskip-closed-tests@57e48d3619bf44b9307a3197239d398b0287b25c" + GH_AW_INFO_BODY_MODIFIED: "false" GH_AW_COMPILED_STRICT: "true" uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -268,21 +274,13 @@ jobs: GH_AW_ACTIONS_DIR: ${{ runner.temp }}/gh-aw/actions GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl - GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"file\":\"safe_outputs_create_pull_request.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0006\"}]}" - GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0006\"}]}" GH_AW_PROMPT_CONTENT_0000: "\n" - GH_AW_PROMPT_CONTENT_0001: "\nTools: create_pull_request, missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: missing_tool, missing_data, noop, apply_verified_unskips\n" GH_AW_PROMPT_CONTENT_0002: "\n" - GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" - GH_AW_PROMPT_CONTENT_0004: "\n" - GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/shared/repo-build-setup.md}}\n" + GH_AW_PROMPT_CONTENT_0003: "\n" + GH_AW_PROMPT_CONTENT_0004: "{{#runtime-import .github/workflows/unskip-closed-tests-prepare.md}}\n" + GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/unskip-closed-tests-shared.md}}\n" GH_AW_PROMPT_CONTENT_0006: "{{#runtime-import .github/workflows/unskip-closed-tests.md}}\n" with: script: | @@ -295,8 +293,6 @@ jobs: env: GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} with: script: | const path = require('path'); @@ -309,15 +305,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt - GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} - GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_MCP_CLI_SERVERS_LIST: "- `github` β€” run `github --help` to see available tools\n- `safeoutputs` β€” run `safeoutputs --help` to see available tools" + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` β€” run `safeoutputs --help` to see available tools' GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} with: script: | @@ -332,14 +320,6 @@ jobs: return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, substitutions: { - GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, - GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, - GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, - GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED } @@ -385,8 +365,11 @@ jobs: retention-days: 1 agent: - needs: activation - if: needs.activation.outputs.daily_ai_credits_exceeded != 'true' + needs: + - activation + - collect-unskip-candidates + if: > + (needs.activation.outputs.daily_ai_credits_exceeded != 'true') && (needs.collect-unskip-candidates.outputs.eligible-count != '0') runs-on: ubuntu-latest permissions: contents: read @@ -448,6 +431,7 @@ jobs: GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/unskip-closed-tests.lock.yml@${{ github.ref }} GH_AW_INFO_VERSION: "1.0.87" GH_AW_INFO_AWF_VERSION: "v0.28.23" + GH_AW_INFO_BODY_MODIFIED: "false" GH_AW_INFO_ENGINE_ID: "copilot" - name: Set runtime paths id: set-runtime-paths @@ -487,10 +471,18 @@ jobs: with: name: activation path: /tmp/gh-aw - - name: Build - run: ./build.sh - - name: Put dotnet on the path - run: echo "$PWD/.dotnet" >> $GITHUB_PATH + - name: Download trusted candidate manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: .gh-aw/unskip-closed-tests + - env: + GH_AW_MANIFEST_DIGEST_VALUE: ${{ needs.collect-unskip-candidates.outputs.manifest-digest }} + GH_AW_SOURCE_COMMIT_VALUE: ${{ needs.collect-unskip-candidates.outputs.source-commit }} + GH_AW_WORKSPACE_VALUE: ${{ github.workspace }} + name: Export trusted manifest context + run: "{\n echo \"GH_AW_UNSKIP_MANIFEST=${GH_AW_WORKSPACE_VALUE}/.gh-aw/unskip-closed-tests/manifest.json\"\n echo \"GH_AW_UNSKIP_SOURCE_COMMIT=${GH_AW_SOURCE_COMMIT_VALUE}\"\n echo \"GH_AW_UNSKIP_MANIFEST_DIGEST=${GH_AW_MANIFEST_DIGEST_VALUE}\"\n} >> \"$GITHUB_ENV\"\n" + shell: bash - name: Configure Git credentials env: @@ -521,18 +513,6 @@ jobs: GH_AW_COMPILED_VERSION: v0.89.21 - name: Install AWF binary run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.23 --rootless - - name: Determine automatic lockdown mode for GitHub MCP Server - id: determine-automatic-lockdown - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) - env: - GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} - GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} - with: - script: | - const path = require('path'); - const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); - const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); - await determineAutomaticLockdown(github, context, core); - name: Restore agent config folders from base branch if: steps.checkout-pr.outcome == 'success' env: @@ -549,7 +529,7 @@ jobs: GH_AW_SKILL_DIR: ".github/skills" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64 ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0 ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086 ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6 + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.23@sha256:2c78aaba1c108e130e2d6d01e4f2cca334ea04c53e6f258913ac34173fe7e3b2 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.23@sha256:c15c3d1208df10c5b588a3657be53742aa982ae0909d1eb1812525268794ca64 ghcr.io/github/gh-aw-firewall/squid:0.28.23@sha256:02ffc56dd40158223064ef03a78c2d9717473c93723b4e403d455ea6f0b09ae0 ghcr.io/github/gh-aw-mcpg:v0.4.25@sha256:9be0a86220e807a0ecc89e53d7453468f7a53fbc6b3d1efd2299025ffe01d086 ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 - name: Prepare Safe Outputs Directories run: | mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" @@ -560,7 +540,7 @@ jobs: env: GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" - GH_AW_SAFE_OUTPUTS_CONFIG: "{\"create_pull_request\":{\"draft\":true,\"expires\":48,\"labels\":[\"type/automation\",\"type/test-gap\"],\"max\":1,\"max_patch_files\":100,\"max_patch_size\":4096,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\"],\"protected_files_policy\":\"fallback-to-issue\",\"title_prefix\":\"[unskip-tests] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"apply-verified-unskips\":{\"description\":\"Publish the exact read-only verified Ignore-removal package in a fresh authenticated checkout and open at most one draft pull request.\",\"inputs\":{\"candidate_ids_json\":{\"default\":null,\"description\":\"JSON array of exact candidate IDs copied from the manifest.\",\"required\":true,\"type\":\"string\"},\"manifest_digest\":{\"default\":null,\"description\":\"Exact trusted manifest digest.\",\"required\":true,\"type\":\"string\"}}},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" with: script: | const path = require('path'); @@ -573,73 +553,35 @@ jobs: env: GH_AW_TOOLS_META_JSON: | { - "description_suffixes": { - "create_pull_request": " CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"[unskip-tests] \". Labels [\"type/automation\" \"type/test-gap\"] will be automatically added. PRs will be created as drafts." - }, + "description_suffixes": {}, "repo_params": {}, - "dynamic_tools": [] + "dynamic_tools": [ + { + "description": "Publish the exact read-only verified Ignore-removal package in a fresh authenticated checkout and open at most one draft pull request.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "candidate_ids_json": { + "description": "JSON array of exact candidate IDs copied from the manifest.", + "type": "string" + }, + "manifest_digest": { + "description": "Exact trusted manifest digest.", + "type": "string" + } + }, + "required": [ + "candidate_ids_json", + "manifest_digest" + ], + "type": "object" + }, + "name": "apply_verified_unskips" + } + ] } GH_AW_VALIDATION_JSON: | { - "create_pull_request": { - "defaultMax": 1, - "fields": { - "base": { - "type": "string", - "sanitize": true, - "maxLength": 128 - }, - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "branch": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "dependencies": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 256 - }, - "draft": { - "type": "boolean" - }, - "labels": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 - }, - "repo": { - "type": "string", - "maxLength": 256 - }, - "stack_position": { - "optionalPositiveInteger": true - }, - "stack_root": { - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "temporary_id": { - "type": "string", - "pattern": "^#?aw_[A-Za-z0-9_]{3,12}$" - }, - "title": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, "missing_data": { "defaultMax": 20, "fields": { @@ -730,10 +672,6 @@ jobs: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} - GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} - GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} - GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eo pipefail @@ -771,26 +709,9 @@ jobs: mkdir -p "$HOME/.copilot" GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_6e47705499a41fbb_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_763bc2030deabb7d_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { - "github": { - "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v1.12.2", - "env": { - "GITHUB_FEATURES": "fields_param", - "GITHUB_HOST": "${GITHUB_SERVER_URL}", - "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}", - "GITHUB_READ_ONLY": "1", - "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" - }, - "guard-policies": { - "allow-only": { - "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", - "repos": "$GITHUB_MCP_GUARD_REPOS" - } - } - }, "safeoutputs": { "type": "stdio", "container": "ghcr.io/github/gh-aw-node", @@ -822,14 +743,6 @@ jobs: "GITHUB_TOKEN": "\${GITHUB_TOKEN}", "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}", "RUNNER_TEMP": "\${RUNNER_TEMP}" - }, - "guard-policies": { - "write-sink": { - "accept": [ - "*" - ], - "sink-visibility": "${GH_AW_SINK_VISIBILITY}" - } } } }, @@ -846,7 +759,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_6e47705499a41fbb_EOF + GH_AW_MCP_CONFIG_763bc2030deabb7d_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -873,37 +786,24 @@ jobs: - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool github # --allow-tool safeoutputs # --allow-tool shell(cat) # --allow-tool shell(date) - # --allow-tool shell(dotnet:*) # --allow-tool shell(echo) - # --allow-tool shell(find) - # --allow-tool shell(git add:*) - # --allow-tool shell(git branch:*) - # --allow-tool shell(git checkout:*) - # --allow-tool shell(git commit:*) - # --allow-tool shell(git merge:*) - # --allow-tool shell(git rm:*) - # --allow-tool shell(git status) - # --allow-tool shell(git switch:*) - # --allow-tool shell(git:*) - # --allow-tool shell(github:*) # --allow-tool shell(grep) # --allow-tool shell(head) + # --allow-tool shell(jq) # --allow-tool shell(ls) # --allow-tool shell(printf) # --allow-tool shell(pwd) - # --allow-tool shell(rg) # --allow-tool shell(safeoutputs:*) + # --allow-tool shell(sed) # --allow-tool shell(sort) # --allow-tool shell(tail) # --allow-tool shell(uniq) # --allow-tool shell(wc) # --allow-tool shell(yq) - # --allow-tool write - timeout-minutes: 30 + timeout-minutes: 20 run: | set -o pipefail printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt @@ -962,8 +862,8 @@ jobs: GH_AW_AWF_LOG_FILE=/tmp/gh-aw/agent-stdio.log \ GH_AW_AWF_ATTEMPT_LOG_NAME=copilot \ bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ - awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; GH_AW_TOOL_BINS="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')"; GH_AW_TOOL_BINS="${GH_AW_TOOL_BINS%:}"; export PATH="$PATH${GH_AW_TOOL_BINS:+:}$GH_AW_TOOL_BINS"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner β€” check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(dotnet:*)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(git add:*)'\'' --allow-tool '\''shell(git branch:*)'\'' --allow-tool '\''shell(git checkout:*)'\'' --allow-tool '\''shell(git commit:*)'\'' --allow-tool '\''shell(git merge:*)'\'' --allow-tool '\''shell(git rm:*)'\'' --allow-tool '\''shell(git status)'\'' --allow-tool '\''shell(git switch:*)'\'' --allow-tool '\''shell(git:*)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(rg)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; GH_AW_TOOL_BINS="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')"; GH_AW_TOOL_BINS="${GH_AW_TOOL_BINS%:}"; export PATH="$PATH${GH_AW_TOOL_BINS:+:}$GH_AW_TOOL_BINS"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner β€” check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sed)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -976,13 +876,12 @@ jobs: GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_TIMEOUT_MINUTES: 30 + GH_AW_TIMEOUT_MINUTES: 20 GH_AW_VERSION: v0.89.21 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md @@ -1000,7 +899,7 @@ jobs: continue-on-error: true env: GH_AW_AGENTIC_EXECUTION_OUTCOME: ${{ steps.agentic_execution.outcome }} - GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: 30 + GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: 20 uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | @@ -1193,12 +1092,343 @@ jobs: /tmp/gh-aw/sandbox/firewall/awf-reflect.json if-no-files-found: ignore + apply_verified_unskips: + needs: + - agent + - detection + - safe_outputs + if: > + (!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'apply_verified_unskips') && + (needs.agent.result == 'success' && needs.detection.result == 'success' && needs.detection.outputs.detection_success == 'true' && + contains(needs.agent.outputs.output_types, 'apply_verified_unskips')) + runs-on: ubuntu-latest + permissions: + contents: write + issues: read + pull-requests: write + steps: + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: ${{ runner.temp }}/gh-aw/safe-jobs/ + - name: Download original trusted manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + with: + name: unskip-closed-tests-manifest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-manifest + - name: Download verified unskip package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + with: + name: unskip-closed-tests-verification-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unskip-closed-tests-verification + - name: Checkout exact analyzed revision with publisher credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + with: + fetch-depth: 0.0 + persist-credentials: true + ref: ${{ github.sha }} + - name: Publish one verified draft pull request + run: | + set -euo pipefail + + test -f "$RESULT_PATH" + test -d "$PACKAGE_DIRECTORY/files" + MANIFEST_DIGEST=$(jq -r '.manifest_digest' "$ORIGINAL_MANIFEST") + test "$MANIFEST_DIGEST" != "null" + jq -e \ + --arg source "$EXPECTED_COMMIT" \ + --arg digest "$MANIFEST_DIGEST" \ + --slurpfile manifest "$ORIGINAL_MANIFEST" \ + '.schema_version == "1" and + .source_commit == $source and + .manifest_digest == $digest and + (.has_changes | type == "boolean") and + if .has_changes then + (.retained_candidates | length) > 0 and + ([.retained_candidates[].candidate_id] | length) == + ([.retained_candidates[].candidate_id] | unique | length) and + all(.retained_candidates[]; + . as $retained | + any($manifest[0].candidates[]; + .candidate_id == $retained.candidate_id and + .path == $retained.path and + .decision.eligible == true and + ((.owner.test_fqns | sort) == ($retained.test_fqns | sort)))) and + (.changed_files | length) > 0 and + ([.changed_files[].path] | length) == + ([.changed_files[].path] | unique | length) and + ([.changed_files[].path] | sort) == (.changed_paths | sort) and + ([.retained_candidates[].path] | unique | sort) == (.changed_paths | sort) and + all(.changed_files[]; + (.path | type == "string") and + (.content_sha256 | type == "string" and test("^[0-9a-f]{64}$"))) and + (.pr_title | type == "string" and + startswith("[unskip-closed-tests] ") and length <= 256) and + (.pr_body | type == "string" and + startswith("")) + else + (.retained_candidates | length) == 0 and + (.changed_files | length) == 0 and + (.changed_paths | length) == 0 and + .pr_title == "" and + .pr_body == "" + end' \ + "$RESULT_PATH" >/dev/null + + if [ "$(jq -r '.has_changes' "$RESULT_PATH")" = "false" ]; then + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -print -quit)" + git diff --quiet + echo "::notice::No selected candidate passed verification." + exit 0 + fi + + DEFAULT_BRANCH=$(gh api "repos/${EXPECTED_REPOSITORY}" --jq '.default_branch') + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + test "$CURRENT_HEAD" = "$EXPECTED_COMMIT" || + { echo "::notice::Default branch advanced; leaving verified changes unpublished."; exit 0; } + + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + test "$EXISTING" -eq 0 || + { echo "::notice::An unskip pull request is already open."; exit 0; } + + TITLE=$(jq -r '.pr_title' "$RESULT_PATH") + BODY_FILE="$RUNNER_TEMP/unskip-closed-tests-pr-body.md" + EXPECTED_FILES="$RUNNER_TEMP/unskip-closed-tests-expected-files.bin" + EXPECTED_PATHS="$RUNNER_TEMP/unskip-closed-tests-expected-paths.bin" + ACTUAL_PATHS="$RUNNER_TEMP/unskip-closed-tests-actual-paths.bin" + EXPECTED_BLOBS="$RUNNER_TEMP/unskip-closed-tests-expected-blobs.txt" + ACTUAL_BLOBS="$RUNNER_TEMP/unskip-closed-tests-actual-blobs.txt" + jq -r '.pr_body' "$RESULT_PATH" > "$BODY_FILE" + jq -j '.changed_files[] | .path, "\u0000", .content_sha256, "\u0000"' \ + "$RESULT_PATH" > "$EXPECTED_FILES" + jq -j '.changed_files[].path, "\u0000"' "$RESULT_PATH" | sort -z > "$EXPECTED_PATHS" + jq -r '[.changed_files[].content_sha256] | unique | sort | .[]' \ + "$RESULT_PATH" > "$EXPECTED_BLOBS" + find "$PACKAGE_DIRECTORY/files" -mindepth 1 -maxdepth 1 -type f \ + -printf '%f\n' | sort > "$ACTUAL_BLOBS" + cmp "$EXPECTED_BLOBS" "$ACTUAL_BLOBS" + test -z "$(find "$PACKAGE_DIRECTORY/files" -mindepth 1 -not -type f -print -quit)" + + git diff --cached --quiet + while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do + test -n "$path" + test "${path#/}" = "$path" + case "/$path/" in + *"/../"*|*"/./"*) exit 20 ;; + esac + case "$path" in + *.cs) ;; + *) echo "::error::Unexpected changed path: $path"; exit 20 ;; + esac + git ls-files --error-unmatch -- "$path" >/dev/null + test -f "$path" + test ! -L "$path" + SOURCE_SHA=$(jq -er \ + --arg path "$path" \ + '[.candidates[] | select(.path == $path) | .source_sha256] | + unique | select(length == 1) | .[0]' \ + "$ORIGINAL_MANIFEST") + ACTUAL_SOURCE_SHA=$(sha256sum -- "$path") + ACTUAL_SOURCE_SHA=${ACTUAL_SOURCE_SHA%% *} + test "$ACTUAL_SOURCE_SHA" = "$SOURCE_SHA" || + { echo "::error::Source content changed for $path"; exit 20; } + BLOB="$PACKAGE_DIRECTORY/files/$expected_sha" + test -f "$BLOB" + test ! -L "$BLOB" + ACTUAL_BLOB_SHA=$(sha256sum -- "$BLOB") + ACTUAL_BLOB_SHA=${ACTUAL_BLOB_SHA%% *} + test "$ACTUAL_BLOB_SHA" = "$expected_sha" || + { echo "::error::Verified package content changed for $path"; exit 20; } + cp -- "$BLOB" "$path" + git add -- "$path" + done < "$EXPECTED_FILES" + + git diff --quiet + git diff --cached --name-only --no-renames -z | sort -z > "$ACTUAL_PATHS" + cmp "$EXPECTED_PATHS" "$ACTUAL_PATHS" + while IFS= read -r -d '' path && IFS= read -r -d '' expected_sha; do + staged_sha=$(git show ":$path" | sha256sum) + staged_sha=${staged_sha%% *} + test "$staged_sha" = "$expected_sha" || + { echo "::error::Staged content does not match verified content for $path"; exit 20; } + done < "$EXPECTED_FILES" + + rm -rf .github/workflows/unskip-closed-tests-tool/bin \ + .github/workflows/unskip-closed-tests-tool/obj + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Re-enable tests with resolved tracking items" + + BRANCH="automation/unskip-closed-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + git push origin "HEAD:refs/heads/$BRANCH" + + CURRENT_HEAD=$(gh api "repos/${EXPECTED_REPOSITORY}/commits/${DEFAULT_BRANCH}" --jq '.sha') + if [ "$CURRENT_HEAD" != "$EXPECTED_COMMIT" ]; then + git push origin --delete "$BRANCH" + echo "::notice::Default branch advanced before PR creation; removed the unpublished branch." + exit 0 + fi + EXISTING=$(gh pr list \ + --repo "$EXPECTED_REPOSITORY" \ + --state open \ + --search 'in:title "[unskip-closed-tests]"' \ + --json number \ + --jq 'length') + if [ "$EXISTING" -ne 0 ]; then + git push origin --delete "$BRANCH" + echo "::notice::Another unskip pull request opened; removed the duplicate branch." + exit 0 + fi + + PR_URL=$(gh pr create \ + --repo "$EXPECTED_REPOSITORY" \ + --base "$DEFAULT_BRANCH" \ + --head "$BRANCH" \ + --draft \ + --title "$TITLE" \ + --body-file "$BODY_FILE") + + LIVE=$(gh pr view "$PR_URL" \ + --repo "$EXPECTED_REPOSITORY" \ + --json baseRefName,baseRefOid,body,isDraft,headRefName,number,state,url) + test "$(printf '%s' "$LIVE" | jq -r '.isDraft')" = "true" + test "$(printf '%s' "$LIVE" | jq -r '.headRefName')" = "$BRANCH" + test "$(printf '%s' "$LIVE" | jq -r '.baseRefName')" = "$DEFAULT_BRANCH" + printf '%s' "$LIVE" | jq -r '.body' | grep -F '