From 7ca2a0b150914e17f925262cb1d6d4bbb7bcb8f7 Mon Sep 17 00:00:00 2001 From: Exoridus Date: Fri, 25 Sep 2026 07:43:15 +0200 Subject: [PATCH] fix(release-verify): bound guest campaign watching and never fall back to a credential prompt --- docs/dev/release-verify-vm.md | 13 +++ scripts/tests/vm-recipe.tests.ps1 | 62 ++++++++++++++ tools/vm/ReleaseVm.psm1 | 136 ++++++++++++++++++++++++++++++ tools/vm/Watch-ReleaseVmRun.ps1 | 74 ++++++++++++++++ 4 files changed, 285 insertions(+) create mode 100644 tools/vm/Watch-ReleaseVmRun.ps1 diff --git a/docs/dev/release-verify-vm.md b/docs/dev/release-verify-vm.md index 64fbd336..1069c1cc 100644 --- a/docs/dev/release-verify-vm.md +++ b/docs/dev/release-verify-vm.md @@ -78,6 +78,19 @@ The runner creates a differencing disk/VM, configures the declared partition/net No campaign receives reusable production credentials. The recipe's local administrator credentials belong only to this disposable, non-RDP guest and must not be reused elsewhere. +## Watch a running campaign + +A campaign that waits on an operator in the guest console gives the host no signal until it ends. Follow it with the watch script instead of an improvised polling loop: + +```powershell +pwsh -NoProfile -NonInteractive -File tools/vm/Watch-ReleaseVmRun.ps1 -VMName '' ` + -ExitFile 'C:\ExoSnapRun\out\' -LogFile 'C:\ExoSnapRun\out\' -TimeoutMinutes 60 +``` + +The watch only reads over PowerShell Direct and never prompts. Each read has its own timeout, repeated failed reads end it with exit code 3 and the deadline ends it with exit code 2. Exit code 0 means the exit file appeared; its content is printed, not propagated. Ending the watch does not stop the campaign. + +Call the module with `pwsh`, never Windows PowerShell. The execution policy stops `powershell.exe` from loading `ReleaseVm.psm1`. The credential is then `$null`, and `Invoke-Command -Credential $null` opens an interactive logon dialog on the host desktop. + ## Limits and recovery Do not use checkpoints, live migration or saved state as a substitute for the differencing-disk model on a GPU-partitioned VM. Do not substitute an RDP desktop for the qualified interactive console. Do not infer general capture support from the availability of a virtual adapter. diff --git a/scripts/tests/vm-recipe.tests.ps1 b/scripts/tests/vm-recipe.tests.ps1 index a1db3785..6e4adb84 100644 --- a/scripts/tests/vm-recipe.tests.ps1 +++ b/scripts/tests/vm-recipe.tests.ps1 @@ -1946,6 +1946,68 @@ Test-Case 'a resumed gpu phase stops the machine itself' { Assert-Equal 'stop-for-gpu' $names[0] 'the phase cannot attach a partition to a running machine' Assert-Match 'Stop-ReleaseVmIfRunning' $plan[0].Command 'stopping must tolerate a machine that is already off' } + +function New-WatchClock { + $clock = [pscustomobject]@{ Time = [DateTime]::new(2026, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) } + return $clock +} + +Test-Case 'a guest watch returns the exit file and every log line in order' { + $reads = [System.Collections.Generic.Queue[object]]::new() + $reads.Enqueue([pscustomobject]@{ Text = "one`n"; Offset = 4; Done = $false; ExitText = $null }) + $reads.Enqueue([pscustomobject]@{ Text = ''; Offset = 4; Done = $false; ExitText = $null }) + $reads.Enqueue([pscustomobject]@{ Text = "two`nlast"; Offset = 12; Done = $true; ExitText = '0' }) + $offsets = [System.Collections.Generic.List[long]]::new() + $printed = [System.Text.StringBuilder]::new() + $result = Watch-ReleaseVmGuestRun -Read { param($Offset) $offsets.Add($Offset); $reads.Dequeue() } ` + -OnText { param($Text) [void] $printed.Append($Text) } -Sleep { param($Seconds) } + Assert-Equal 'finished' $result.Outcome 'an exit file ends the watch' + Assert-Equal '0' $result.ExitText 'the exit file content is reported' + Assert-Equal "one`ntwo`nlast" $printed.ToString() 'log text is printed once, in order' + Assert-Equal '0 4 4' ($offsets -join ' ') 'each read continues at the offset the previous one returned' +} + +Test-Case 'a guest watch that never sees an exit file ends at its deadline' { + $clock = New-WatchClock + $result = Watch-ReleaseVmGuestRun -TimeoutMinutes 1 -PollSeconds 15 -Now { $clock.Time } ` + -Sleep { param($Seconds) $clock.Time = $clock.Time.AddSeconds($Seconds) } ` + -Read { param($Offset) [pscustomobject]@{ Text = ''; Offset = 0; Done = $false; ExitText = $null } } ` + -OnText { param($Text) } + Assert-Equal 'deadline' $result.Outcome 'an operator gate that is never answered must not hold the host' +} + +Test-Case 'a guest watch gives up after consecutive failed reads, and a good read resets the count' { + $script:watchCalls = 0 + $result = Watch-ReleaseVmGuestRun -MaxConsecutiveFailures 3 -Sleep { param($Seconds) } -OnText { param($Text) } ` + -Read { + param($Offset) + $script:watchCalls++ + if ($script:watchCalls -eq 3) { + return [pscustomobject]@{ Text = ''; Offset = 0; Done = $false; ExitText = $null } + } + throw 'guest gone' + } + Assert-Equal 'unreachable' $result.Outcome 'a guest that stops answering ends the watch' + Assert-Equal 6 $script:watchCalls 'two failures, one success, then three failures' + Assert-Match 'guest gone' $result.Detail 'the last failure is named' +} + +Test-Case 'a progress read with no credential is refused instead of prompting' { + $refused = $false + try { + Read-ReleaseVmGuestProgress -VMName 'ExoSnap-None' -Credential $null -ExitFile 'C:\x' | Out-Null + } + catch { $refused = $_.Exception.Message -match 'Credential' } + Assert-True $refused 'a null credential reaching Invoke-Command opens a logon dialog on the host' +} + +Test-Case 'the watch script refuses a missing argument without asking for it' { + $run = Invoke-Script -Path (Join-Path $script:VmRoot 'Watch-ReleaseVmRun.ps1') -Arguments @('-VMName', 'x') + Assert-Equal 1 $run.ExitCode 'a missing exit file is a usage failure' + Assert-Match '-ExitFile is required' $run.Output 'the refusal names the argument' + $source = Get-Content -LiteralPath (Join-Path $script:VmRoot 'Watch-ReleaseVmRun.ps1') -Raw + Assert-NoMatch '\[Parameter\(Mandatory' $source 'a mandatory parameter prompts when a host is interactive' +} Write-Host '' Write-Host " $($script:Passed) passed, $($script:Failed) failed." if ($script:Failed -gt 0) { exit 1 } diff --git a/tools/vm/ReleaseVm.psm1 b/tools/vm/ReleaseVm.psm1 index 69d26462..1e6b67eb 100644 --- a/tools/vm/ReleaseVm.psm1 +++ b/tools/vm/ReleaseVm.psm1 @@ -2783,6 +2783,140 @@ function Invoke-ReleaseVmCommand { return $result } +function Read-ReleaseVmGuestProgress { + <# + .SYNOPSIS + One bounded read of a running guest campaign: new log lines and whether it finished. + .DESCRIPTION + The log is read from a byte offset and only complete lines come back, so a line + the campaign is still writing arrives whole on a later read. Once the exit file + exists the remainder is returned as is. The log is expected to be UTF-8; a + rewritten or truncated log restarts at its beginning. + + The read runs as a job with its own timeout because a PowerShell Direct call + into a guest that stopped answering has no transport timeout. A timed-out job + is abandoned rather than stopped, since stopping it can block just as long; it + ends with the calling process. + #> + param( + [Parameter(Mandatory)] [string] $VMName, + [Parameter(Mandatory)] [System.Management.Automation.PSCredential] $Credential, + [Parameter(Mandatory)] [string] $ExitFile, + [string] $LogFile, + [long] $Offset = 0, + [ValidateRange(5, 600)] [int] $TimeoutSeconds = 60, + [ValidateRange(1024, 16777216)] [int] $MaxBytes = 262144 + ) + # The guest side runs in the guest's Windows PowerShell 5.1. + $job = Invoke-Command -VMName $VMName -Credential $Credential -AsJob -ScriptBlock { + param($ExitPath, $LogPath, $From, $Limit) + $done = Test-Path -LiteralPath $ExitPath + $exitText = $null + if ($done) { $exitText = "$(Get-Content -LiteralPath $ExitPath -Raw)".Trim() } + + $text = '' + $next = $From + if ($LogPath -and (Test-Path -LiteralPath $LogPath)) { + $stream = [IO.File]::Open($LogPath, [IO.FileMode]::Open, [IO.FileAccess]::Read, + [IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete) + try { + if ($stream.Length -lt $From) { $From = 0 } + $count = [int][Math]::Min($stream.Length - $From, $Limit) + $bytes = New-Object byte[] $count + [void] $stream.Seek($From, [IO.SeekOrigin]::Begin) + $read = 0 + while ($read -lt $count) { + $n = $stream.Read($bytes, $read, $count - $read) + if ($n -le 0) { break } + $read += $n + } + $take = $read + if (-not $done -and $read -gt 0 -and $read -lt $Limit) { + $take = [Array]::LastIndexOf($bytes, [byte] 10, $read - 1) + 1 + } + $skip = 0 + if ($From -eq 0 -and $take -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) { + $skip = 3 + } + $text = [Text.Encoding]::UTF8.GetString($bytes, $skip, $take - $skip) + $next = $From + $take + } + finally { + $stream.Dispose() + } + } + [pscustomobject]@{ Text = $text; Offset = $next; Done = $done; ExitText = $exitText } + } -ArgumentList $ExitFile, $LogFile, $Offset, $MaxBytes + + if (-not (Wait-Job -Job $job -Timeout $TimeoutSeconds)) { + throw "'$VMName' did not answer a progress read within $TimeoutSeconds second(s)" + } + try { + $result = Receive-Job -Job $job -ErrorAction Stop + } + finally { + Remove-Job -Job $job -Force + } + if ($null -eq $result) { throw "'$VMName' returned no progress" } + return [pscustomobject]@{ + Text = [string] $result.Text; Offset = [long] $result.Offset + Done = [bool] $result.Done; ExitText = $result.ExitText + } +} + +function Watch-ReleaseVmGuestRun { + <# + .SYNOPSIS + Follows a guest campaign until it finishes, a deadline passes or the guest stops answering. + .DESCRIPTION + Every way out is bounded. Each read carries its own timeout, a run of failed + reads ends the watch as unreachable, and the deadline ends it whatever the guest + does. A campaign waiting on an operator shows as a quiet log, never as a host + that hangs. The watch only reads: it neither prompts nor changes the guest. + + -Read receives the next offset and returns an object with Text, Offset, Done and + ExitText, as Read-ReleaseVmGuestProgress does. New text is passed to -OnText. + The result's Outcome is 'finished', 'deadline' or 'unreachable'. + #> + param( + [Parameter(Mandatory)] [scriptblock] $Read, + [Parameter(Mandatory)] [scriptblock] $OnText, + [ValidateRange(1, 1440)] [int] $TimeoutMinutes = 90, + [ValidateRange(1, 600)] [int] $PollSeconds = 15, + [ValidateRange(1, 100)] [int] $MaxConsecutiveFailures = 4, + [scriptblock] $Now = { [DateTime]::UtcNow }, + [scriptblock] $Sleep = { param($Seconds) Start-Sleep -Seconds $Seconds } + ) + $deadline = (& $Now).AddMinutes($TimeoutMinutes) + $offset = [long] 0 + $failures = 0 + while ((& $Now) -lt $deadline) { + try { + $progress = & $Read $offset + $failures = 0 + if ($progress.Text) { & $OnText $progress.Text } + $offset = [long] $progress.Offset + if ($progress.Done) { + return [pscustomobject]@{ Outcome = 'finished'; ExitText = $progress.ExitText; Detail = '' } + } + } + catch { + $failures++ + if ($failures -ge $MaxConsecutiveFailures) { + return [pscustomobject]@{ + Outcome = 'unreachable'; ExitText = $null + Detail = "$failures consecutive reads failed; last: $($_.Exception.Message)" + } + } + } + & $Sleep $PollSeconds + } + return [pscustomobject]@{ + Outcome = 'deadline'; ExitText = $null + Detail = "no exit file within $TimeoutMinutes minute(s); the campaign may still be running" + } +} + function Copy-ReleaseVmDirectoryBack { <# .SYNOPSIS @@ -2914,4 +3048,6 @@ Export-ModuleMember -Function @( 'Assert-ReleaseVmSealed' 'Invoke-ReleaseVmProvisioning' 'Invoke-ReleaseVmCommand' + 'Read-ReleaseVmGuestProgress' + 'Watch-ReleaseVmGuestRun' ) diff --git a/tools/vm/Watch-ReleaseVmRun.ps1 b/tools/vm/Watch-ReleaseVmRun.ps1 new file mode 100644 index 00000000..1f9e0832 --- /dev/null +++ b/tools/vm/Watch-ReleaseVmRun.ps1 @@ -0,0 +1,74 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS + Follows a campaign running in a release-verification guest, read-only and bounded. + Runs on the HOST. + +.DESCRIPTION + Prints the guest campaign's log as it grows and returns once the guest writes its + exit file. The watch never waits without a limit: each read over PowerShell Direct + has its own timeout, repeated failed reads end it, and -TimeoutMinutes ends it + regardless. It never prompts. A missing argument is an error, not a question, so + it is safe to run unattended or from a background monitor. + + Exit codes: 0 the exit file appeared (its content is printed, not propagated), + 2 the deadline passed, 3 the guest stopped answering, 1 anything else. + +.PARAMETER VMName + The running guest to read from. + +.PARAMETER ExitFile + Guest path the campaign writes when it is done. + +.PARAMETER LogFile + Guest path of a UTF-8 log to follow. Optional. + +.EXAMPLE + pwsh -NoProfile -NonInteractive -File tools/vm/Watch-ReleaseVmRun.ps1 -VMName ExoSnap-Run-smoke-001 ` + -ExitFile $guestExitFile -LogFile $guestLogFile -TimeoutMinutes 60 +#> +[CmdletBinding()] +param( + # Not Mandatory: PowerShell asks for a missing mandatory argument, and a question + # nobody sees stalls an unattended watch. + [string] $VMName, + [string] $ExitFile, + [string] $LogFile, + [ValidateRange(1, 1440)] [int] $TimeoutMinutes = 90, + [ValidateRange(1, 600)] [int] $PollSeconds = 15, + [ValidateRange(1, 100)] [int] $MaxConsecutiveFailures = 4 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$code = 1 +try { + if (-not $VMName) { throw '-VMName is required' } + if (-not $ExitFile) { throw '-ExitFile is required' } + + Import-Module (Join-Path $PSScriptRoot 'ReleaseVm.psm1') -Force -DisableNameChecking + $credential = New-ReleaseVmCredential + $read = { + param($Offset) + Read-ReleaseVmGuestProgress -VMName $VMName -Credential $credential -ExitFile $ExitFile ` + -LogFile $LogFile -Offset $Offset + }.GetNewClosure() + $print = { param($Text) [Console]::Out.Write($Text); [Console]::Out.Flush() } + + $result = Watch-ReleaseVmGuestRun -Read $read -OnText $print -TimeoutMinutes $TimeoutMinutes ` + -PollSeconds $PollSeconds -MaxConsecutiveFailures $MaxConsecutiveFailures + switch ($result.Outcome) { + 'finished' { [Console]::Out.WriteLine("campaign finished; exit file: $($result.ExitText)"); $code = 0 } + 'deadline' { [Console]::Out.WriteLine("watch ended: $($result.Detail)"); $code = 2 } + 'unreachable' { [Console]::Out.WriteLine("watch ended, '$VMName' unreachable: $($result.Detail)"); $code = 3 } + } +} +catch { + [Console]::Error.WriteLine("watch failed: $($_.Exception.Message)") + $code = 1 +} +[Console]::Out.Flush() +# A normal exit disposes the runspaces of abandoned PowerShell Direct reads, and that +# blocks for as long as the guest does not answer. +[Environment]::Exit($code)