diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 3fc1c2ce..261b74ae 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -9,13 +9,6 @@ ], "rollForward": false }, - "dotnet-inspect": { - "version": "0.26.0", - "commands": [ - "dotnet-inspect" - ], - "rollForward": false - }, "purview.build": { "version": "0.8.0", "commands": [ diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 02b061cd..af181942 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -50,14 +50,52 @@ jobs: - name: Test SampleApp run: dotnet test samples/SampleApp/SampleApp.slnx --configuration Release --no-build + # Proving that each ForbiddenContent rule bites is cheap - pack the generator, then validate injected copies of it - + # so it runs in its own job rather than as the last step of `packed-consumers`. There it could only fail after the + # Native AOT and WebAssembly AOT publishes had already spent seven minutes. + # + # Like `samples`, it needs no feed wiring of its own: the shared `build` job registers the organisation's GitHub + # Packages source when a pinned version is not on nuget.org yet, and the pack below fails in seconds, naming the + # package and version, if one cannot be restored. + pack-validation: + name: Pack validation rule proof + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + # Restore before pack: a pwsh step that left `dotnet tool restore`'s non-zero exit code to the shell's + # trailing `exit $LASTEXITCODE` is what made this gate report a green run as a failure. Reaching a step at + # all must mean it succeeded, so the failure is raised here, with the message above it. + - name: Pack the generator and validate its content + shell: pwsh + run: | + dotnet tool restore + if ($LASTEXITCODE -ne 0) { + throw 'dotnet tool restore failed; the message above names the command and the package it is missing from.' + } + dotnet purview-build ` + --Build:RunPack=true ` + --Build:ValidatePack=true ` + --Build:RunTests=false ` + --Build:RunLint=false ` + --Release:Mode=None + + - name: Prove every pack-validation forbidden-content rule rejects a package + shell: pwsh + run: ./scripts/test-pack-validation-rules.ps1 -PackagesFolder artifacts + packed-consumers: name: Packed consumers (Native AOT, Blazor WebAssembly, Blazor Server) runs-on: ubuntu-latest timeout-minutes: 45 - env: - # An extra feed for Purview.SourceGeneratorFramework and Purview.BuildSdk only, for versions - # that are not on nuget.org yet. Unset, the gate restores them from nuget.org alone. - TELEMETRY_PRERELEASE_FEED: ${{ vars.TELEMETRY_PRERELEASE_FEED }} steps: - uses: actions/checkout@v7 with: @@ -71,8 +109,7 @@ jobs: # First, before any toolchain is installed: the generator is built against the # Purview.SourceGeneratorFramework version Directory.Packages.props pins, with the # Purview.BuildSdk version global.json pins, so nothing can be packed until both restore. - # This fails with what to do (publish them to nuget.org, or set the - # TELEMETRY_PRERELEASE_FEED repository variable) rather than skipping. + # This fails with what to do rather than skipping. - name: Check the prerelease build dependencies restore shell: pwsh run: ./scripts/test-packed-consumers.ps1 -PreflightOnly @@ -91,9 +128,3 @@ jobs: run: >- ./scripts/test-packed-consumers.ps1 -RuntimeIdentifier linux-x64 -WasmAot Require -PackOutput artifacts/packed - - - name: Prove every pack-validation forbidden-content rule rejects a package - shell: pwsh - run: | - dotnet tool restore - ./scripts/test-pack-validation-rules.ps1 -PackagesFolder artifacts/packed diff --git a/AGENTS.md b/AGENTS.md index 0f9f05a2..a3ea7400 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,11 +165,11 @@ Treat work as incomplete until every gate relevant to it passes. Before a releas emitted activities, metrics and log entries, log-level gating and multi-target methods), a Blazor WebAssembly app (trimmed and `RunAOTCompilation`; no generated member trimmed away, every generated method AOT-compiled) and a Blazor Server host (page rendered on a `TestServer`, telemetry asserted). - Every consumer's bin and publish output is leak-scanned. Set `LOCAL_NUGET_FEED_PATH` (or - `TELEMETRY_PRERELEASE_FEED`) while Purview.SourceGeneratorFramework or Purview.BuildSdk is not on - nuget.org yet. It needs the Native AOT toolchain and, for the WebAssembly AOT step, the `wasm-tools` - workload. -- `just verify-pack-rules` after any change to `PackValidation` in `.config/purview-build.json`. + Every consumer's bin and publish output is leak-scanned. Set `LOCAL_NUGET_FEED_PATH` while + Purview.SourceGeneratorFramework or Purview.BuildSdk is not on nuget.org yet. It needs the Native AOT + toolchain and, for the WebAssembly AOT step, the `wasm-tools` workload. +- `just verify-pack-rules` after any change to `PackValidation` in `.config/purview-build.json` (it is also the last + step of `just pipeline-pack-validate`, so packing locally proves the rules without a second command). - `just build-s && just test-s` — the samples, which reference the generator as an analyzer project. - `just lint` — CSharpier reports no change (`just lint-fix` to apply it). diff --git a/Justfile b/Justfile index 4ff1cc3b..50b1d436 100644 --- a/Justfile +++ b/Justfile @@ -69,8 +69,8 @@ smoke-packed-consumers *args: # Prove every PackValidation ForbiddenContent rule in .config/purview-build.json rejects a package: injects a matching # file into scratch copies of the packed package and runs the real purview-build validation on them. Reads artifacts/, -# so pack first (`just pipeline-pack-validate`). `just pipeline-local-release` moves its packages to the local feed, so -# pass -PackagesFolder p:/_sync-projects/.local-nuget after it. +# so pack first (`just pipeline-pack-validate`, which runs this recipe itself). `just pipeline-local-release` moves its +# packages to the local feed, so pass -PackagesFolder p:/_sync-projects/.local-nuget after it. [group('Pipeline')] verify-pack-rules *args: echo "Proving the pack-validation forbidden-content rules..." @@ -83,12 +83,15 @@ pipeline-tests *args: echo "Running tests pipeline..." {{ pipeline_tool }} --Build:RunTests=true --Release:Mode=None {{ args }} -# Run the pipeline through pack + validate (restore, build, lint, tests, pack, validate pack contents) without publishing/releasing +# Run the pipeline through pack + validate (restore, build, lint, tests, pack, validate pack contents) without +# publishing/releasing. Finishes by proving every ForbiddenContent rule rejects a package (`verify-pack-rules`, which +# reads artifacts/ - pass -Build:ArtifactsFolder to that recipe if args moved this run's packages somewhere else). [group('Pipeline')] pipeline-pack-validate *args: just ensure-pipeline-tool echo "Running pack + validate pipeline..." {{ pipeline_tool }} --Build:RunPack=true --Build:ValidatePack=true --Release:Mode=None {{ args }} + just verify-pack-rules # ----------------------------------------------------------------------------- # Build and Test diff --git a/docs/wiki/Release-Flow.md b/docs/wiki/Release-Flow.md index 092aec0d..642c3a02 100644 --- a/docs/wiki/Release-Flow.md +++ b/docs/wiki/Release-Flow.md @@ -49,24 +49,27 @@ check fails rather than passing. | Gate | What it proves | Local | CI | | --- | --- | --- | --- | | Build, lint and tests | The solution builds warning-free, CSharpier reports no change, and every test project passes. | `just build`, `just lint`, `just test` | `pr.yml` `build` | -| Pack and pack validation | The package packs; `purview-build.json` `PackValidation` finds exactly the declared content (`RequiredContent`, `RequireExplicitContent`) and none of the `ForbiddenContent`. | `just pipeline-pack-validate` | `pr.yml` `build` | -| Forbidden-content rule proof | Every `ForbiddenContent` rule rejects a package: a matching file is injected into scratch copies of the package and the real `purview-build` validation must fail, per package and per rule. | `just verify-pack-rules` | `pr.yml` `packed-consumers` | +| Pack and pack validation | The package packs; `purview-build.json` `PackValidation` finds exactly the declared content (`RequiredContent`, `RequireExplicitContent`) and none of the `ForbiddenContent`. | `just pipeline-pack-validate` (which also runs the rule proof) | `pr.yml` `build` | +| Forbidden-content rule proof | Every `ForbiddenContent` rule rejects a package: a matching file is injected into scratch copies of the package and the real `purview-build` validation must fail, per package and per rule. | `just verify-pack-rules` (and the end of `just pipeline-pack-validate`) | `pr.yml` `pack-validation`, in parallel with the consumer gate | | Packed consumers | The package, packed to an isolated feed and cache, works for consumers that reference it as a package - each with a class library of its own generated telemetry. (a) A **Native AOT** console publishes with `-warnaserror` and no IL2xxx/IL3xxx warning, then runs with an in-process `ActivityListener`, `MeterListener` and capturing logger provider and checks the multi-target method (Activity, Information log and counter from one call), tags, baggage, events and the OpenTelemetry exception event, every metric measurement, v1 and v2 logging, log-level gating and the `TelemetryNames` aggregation. (b) A **Blazor WebAssembly** app, using the telemetry from a page and a service, publishes trimmed with `-warnaserror` and with `RunAOTCompilation=true`: the trimmed assemblies keep every generated member, and the WebAssembly AOT compiler compiles every generated method. (c) A **Blazor Server** host renders its page on an in-memory `TestServer`, and the telemetry the page emitted is checked as in (a). (d) Every bin and publish folder is leak-scanned. | `just smoke-packed-consumers` | `pr.yml` `packed-consumers` (linux-x64, `-WasmAot Require`) | | Compiler-host leak scan | No generator, refactorings, Purview.SourceGeneratorFramework or Roslyn assembly reaches build or publish output, by file name or in a `deps.json`, `runtimeconfig.json` or boot manifest (`blazor.boot.json`, `dotnet.boot.js`, `dotnet.js`). Purview.SourceGeneratorFramework 1.2.1 put its DLL in every sample's `bin` while every build passed. | `just scan-compiler-host-leaks ...` | the `samples` job (every sample's `bin`) and every consumer in `packed-consumers`; `ReleaseGateTests` fails a job that builds the samples or publishes without it | `ReleaseGateTests` (in `src/tests/Repository.UnitTests`, so `just test` runs it) guards the gates themselves: the leak scan must catch each kind of leak it claims to, every workflow job that builds the samples or publishes must scan its -output, the `packed-consumers` job must run the gate and the rule proof, the Justfile recipes must run their scripts, -the consumers must reference the generator as a package and never the repository's projects, and `ForbiddenContent` -must keep every rule. +output, the `packed-consumers` job must run the gate and the rule proof must have a cheap job of its own, the Justfile +recipes must run their scripts, every gate script must state its exit code rather than inherit a non-zero +`$LASTEXITCODE` (GitHub's `pwsh` shell exits with it, which reported a green proof as a failed step), the consumers +must reference the generator as a package and never the repository's projects, and `ForbiddenContent` must keep every +rule. ### Prerelease build dependencies The generator is built against the Purview.SourceGeneratorFramework version `Directory.Packages.props` pins, with the -Purview.BuildSdk version `global.json` pins. Until those versions are on nuget.org, CI cannot restore them: the -`packed-consumers` job fails at its first step with what to do, rather than skipping. Publish them, or set the -`TELEMETRY_PRERELEASE_FEED` repository variable to a feed that has them. Locally, `just smoke-packed-consumers` also -reads `TELEMETRY_PRERELEASE_FEED` or `LOCAL_NUGET_FEED_PATH`, and uses that feed for those packages only. +Purview.BuildSdk version `global.json` pins. Until those versions are on nuget.org, nothing can be packed, so the +`packed-consumers` job fails at its first step with what to do, rather than skipping. Publish them to nuget.org, or +rely on the organisation's GitHub Packages source, which the shared `build` workflow registers when required. Locally, +`just smoke-packed-consumers` reads `LOCAL_NUGET_FEED_PATH` (or takes `-PrereleaseFeed`) and uses that feed for those +packages only. The consumers target `net11.0` and take their ASP.NET Core package versions from the `Packed consumers` group in `Directory.Packages.props`, so bump those with the .NET 11 SDK in `global.json`. diff --git a/scripts/test-pack-validation-rules.ps1 b/scripts/test-pack-validation-rules.ps1 index fd161fdf..d98f588f 100644 --- a/scripts/test-pack-validation-rules.ps1 +++ b/scripts/test-pack-validation-rules.ps1 @@ -20,6 +20,10 @@ The packages in -PackagesFolder are never modified. + Every validation run is expected to fail, so $LASTEXITCODE is left non-zero on the way out. The script therefore + exits explicitly: GitHub appends `if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE }` to a + pwsh step, which would otherwise report a green proof as a failed step. + .PARAMETER PackagesFolder Folder holding the packed .nupkg (and any .snupkg) files of the current version. Defaults to artifacts/, where `just pipeline-pack-validate` and `just pack` write them. `just pipeline-local-release` moves them to the local feed, @@ -93,7 +97,13 @@ function Invoke-Validation([string] $name, [System.IO.FileInfo[]] $nupkgs, [stri Push-Location $root try { - & dotnet tool restore | Out-Null + # Fatal, not ignored: a restore that failed silently would leave the runs below using whatever version the + # local tool store already holds, so a stale purview-build could report rules this repository no longer has. + $restore = & dotnet tool restore 2>&1 | ForEach-Object { "$_" } + if ($LASTEXITCODE -ne 0) { + throw "dotnet tool restore failed in '$root', so the pinned purview.build could not be installed:`n$($restore -join "`n")" + } + $output = & dotnet purview-build ` --Build:Solution=empty.slnx ` --Build:ArtifactsFolder=artifacts ` @@ -177,3 +187,6 @@ if ($failures.Count -gt 0) { } Write-Host "Every ForbiddenContent rule ($($cases.Count)) rejected its file, in every package ($($packages.Count))." -ForegroundColor Green + +# See the header: every run above failed on purpose, so the non-zero $LASTEXITCODE must not become the exit code. +exit 0 diff --git a/scripts/test-packed-consumers.ps1 b/scripts/test-packed-consumers.ps1 index 23d99eaa..a8c6ea16 100644 --- a/scripts/test-packed-consumers.ps1 +++ b/scripts/test-packed-consumers.ps1 @@ -56,7 +56,7 @@ .PARAMETER PrereleaseFeed A NuGet feed (folder or URL) to restore Purview.SourceGeneratorFramework and Purview.BuildSdk from, in addition to nuget.org - for versions that are not on nuget.org yet. The feed is used for those packages only. Defaults to - $env:TELEMETRY_PRERELEASE_FEED, then $env:LOCAL_NUGET_FEED_PATH. + $env:LOCAL_NUGET_FEED_PATH. .PARAMETER WasmAot Auto (default) runs the WebAssembly AOT publish when the wasm-tools workload is installed and says so when it is @@ -82,7 +82,7 @@ param ( [string] $Configuration = 'Release', [string] $RuntimeIdentifier = [System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier, - [string] $PrereleaseFeed = ($env:TELEMETRY_PRERELEASE_FEED ? $env:TELEMETRY_PRERELEASE_FEED : $env:LOCAL_NUGET_FEED_PATH), + [string] $PrereleaseFeed = $env:LOCAL_NUGET_FEED_PATH, [ValidateSet('Auto', 'Require', 'Skip')] [string] $WasmAot = 'Auto', [string] $PackOutput, @@ -433,9 +433,9 @@ try { "Purview.SourceGeneratorFramework $sgfVersion (Directory.Packages.props) or Purview.BuildSdk $buildSdkVersion " + "(global.json) could not be restored from $from. The generator is built with both, so nothing can be " + 'packed. If a version is not on nuget.org yet, publish it there, or point this gate at a feed that has ' + - 'it: -PrereleaseFeed , or set TELEMETRY_PRERELEASE_FEED (LOCAL_NUGET_FEED_PATH is used ' + - 'when set). Purview.BuildSdk is resolved by MSBuild through the normal NuGet configuration, so that feed ' + - 'must also be a configured source for the pack itself.' + 'it: -PrereleaseFeed , or set LOCAL_NUGET_FEED_PATH. Purview.BuildSdk is resolved by ' + + 'MSBuild through the normal NuGet configuration, so that feed must also be a configured source for the ' + + 'pack itself.' ) } } @@ -643,3 +643,7 @@ finally { if ($exitCode -ne 0) { exit $exitCode } Write-Host 'Packed-consumer gate passed.' -ForegroundColor Green + +# The last native command was a restore in the finally block; state the exit code rather than inherit it, because +# GitHub's pwsh shell exits with $LASTEXITCODE when a script falls through. +exit 0 diff --git a/src/tests/Repository.UnitTests/ReleaseGateTests.cs b/src/tests/Repository.UnitTests/ReleaseGateTests.cs index dfd065dc..a6523c49 100644 --- a/src/tests/Repository.UnitTests/ReleaseGateTests.cs +++ b/src/tests/Repository.UnitTests/ReleaseGateTests.cs @@ -256,7 +256,7 @@ await Assert } [Test] - public async Task PrWorkflow_GivenThePackedConsumersJob_RunsTheGateAndTheRuleProof( + public async Task PrWorkflow_GivenThePackedConsumersJob_RunsTheGateWithoutTheRuleProof( CancellationToken cancellationToken ) { @@ -272,8 +272,83 @@ CancellationToken cancellationToken // Assert await Assert.That(job).Contains($"./{PackedConsumersScript} -PreflightOnly"); await Assert.That(job).Contains("-WasmAot Require"); - await Assert.That(job).Contains($"./{PackRulesScript}"); await Assert.That(job).Contains("dotnet workload install wasm-tools"); + + // The proof is minutes of work against a package; this job is half an hour of Native AOT and WebAssembly AOT + // publishing. Behind it, a broken rule can only fail after all of that, so the proof has a job of its own. + await Assert + .That(job) + .DoesNotContain(PackRulesScript) + .Because("the rule proof must not sit behind the packed-consumer gate"); + } + + /// + /// The rule proof needs nothing a plain pack does not, so it packs its own package and runs in parallel with the + /// consumer gate. Watching it cost seven minutes of AOT publishing in the pull request that introduced it. + /// + [Test] + public async Task PrWorkflow_GivenThePackValidationJob_ProvesTheRulesFromItsOwnPack( + CancellationToken cancellationToken + ) + { + // Arrange + var text = await File.ReadAllTextAsync( + System.IO.Path.Combine(RepositoryRoot.Value, ".github", "workflows", "pr.yml"), + cancellationToken + ); + + // Act + var job = Jobs(text).SingleOrDefault(static j => j.Name == "pack-validation").Body ?? string.Empty; + + // Assert + await Assert.That(job).IsNotEmpty().Because("pr.yml has no 'pack-validation' job"); + await Assert.That(job).Contains($"./{PackRulesScript}"); + await Assert.That(job).Contains("--Build:RunPack=true").Because("the proof needs packages to inject into"); + await Assert.That(job).Contains("--Build:ValidatePack=true"); + await Assert + .That(job) + .DoesNotContain("dotnet workload install") + .Because("the proof must stay cheap: no Native AOT or WebAssembly toolchain"); + await Assert + .That(job) + .DoesNotContain("dotnet publish") + .Because("the proof must stay cheap: it does not publish a consumer"); + + // A bare `dotnet tool restore` in a pwsh step leaves a non-zero $LASTEXITCODE for GitHub's trailing + // `exit $LASTEXITCODE`, which is how a green step was reported as a failure. Check the code, not the message. + await Assert + .That(job) + .Contains("if ($LASTEXITCODE -ne 0)") + .Because("a failed tool restore must raise, not be inherited by the step's exit code"); + } + + /// + /// GitHub's pwsh shell appends if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE } + /// to every step. These gates run commands that fail on purpose - the rule proof's validations must each be + /// rejected - so a script that falls through reports success as exit code 1. + /// + [Test] + [Arguments(PackedConsumersScript)] + [Arguments(PackRulesScript)] + public async Task GateScript_GivenCommandsThatFailByDesign_EndsWithAnExplicitExit( + string script, + CancellationToken cancellationToken + ) + { + // Arrange + var lines = await File.ReadAllLinesAsync( + System.IO.Path.Combine(RepositoryRoot.Value, script), + cancellationToken + ); + + // Act + var last = lines.Reverse().FirstOrDefault(static line => !string.IsNullOrWhiteSpace(line))?.Trim(); + + // Assert + await Assert + .That(last) + .IsEqualTo("exit 0") + .Because($"{script} must state its exit code rather than leave $LASTEXITCODE to the shell"); } [Test]