fix(release-verify): bound guest campaign watching and never fall back to a credential prompt

docs/dev/release-verify-vm.md

@@ -78,6 +78,19 @@ The runner creates a differencing disk/VM, configures the declared partition/net
 
 No campaign receives reusable production credentials. The recipe's local administrator credentials belong only to this disposable, non-RDP guest and must not be reused elsewhere.
 
+## Watch a running campaign
+
+A campaign that waits on an operator in the guest console gives the host no signal until it ends. Follow it with the watch script instead of an improvised polling loop:
+
+```powershell
+pwsh -NoProfile -NonInteractive -File tools/vm/Watch-ReleaseVmRun.ps1 -VMName '<vm>' `
+    -ExitFile 'C:\ExoSnapRun\out\<exit file>' -LogFile 'C:\ExoSnapRun\out\<log>' -TimeoutMinutes 60
+```
+
+The watch only reads over PowerShell Direct and never prompts. Each read has its own timeout, repeated failed reads end it with exit code 3 and the deadline ends it with exit code 2. Exit code 0 means the exit file appeared; its content is printed, not propagated. Ending the watch does not stop the campaign.
+
+Call the module with `pwsh`, never Windows PowerShell. The execution policy stops `powershell.exe` from loading `ReleaseVm.psm1`. The credential is then `$null`, and `Invoke-Command -Credential $null` opens an interactive logon dialog on the host desktop.
+
 ## Limits and recovery
 
 Do not use checkpoints, live migration or saved state as a substitute for the differencing-disk model on a GPU-partitioned VM. Do not substitute an RDP desktop for the qualified interactive console. Do not infer general capture support from the availability of a virtual adapter.

scripts/tests/vm-recipe.tests.ps1

@@ -1946,6 +1946,68 @@ Test-Case 'a resumed gpu phase stops the machine itself' {
     Assert-Equal 'stop-for-gpu' $names[0] 'the phase cannot attach a partition to a running machine'
     Assert-Match 'Stop-ReleaseVmIfRunning' $plan[0].Command 'stopping must tolerate a machine that is already off'
 }
+
+function New-WatchClock {
+    $clock = [pscustomobject]@{ Time = [DateTime]::new(2026, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) }
+    return $clock
+}
+
+Test-Case 'a guest watch returns the exit file and every log line in order' {
+    $reads = [System.Collections.Generic.Queue[object]]::new()
+    $reads.Enqueue([pscustomobject]@{ Text = "one`n"; Offset = 4; Done = $false; ExitText = $null })
+    $reads.Enqueue([pscustomobject]@{ Text = ''; Offset = 4; Done = $false; ExitText = $null })
+    $reads.Enqueue([pscustomobject]@{ Text = "two`nlast"; Offset = 12; Done = $true; ExitText = '0' })
+    $offsets = [System.Collections.Generic.List[long]]::new()
+    $printed = [System.Text.StringBuilder]::new()
+    $result = Watch-ReleaseVmGuestRun -Read { param($Offset) $offsets.Add($Offset); $reads.Dequeue() } `
+        -OnText { param($Text) [void] $printed.Append($Text) } -Sleep { param($Seconds) }
+    Assert-Equal 'finished' $result.Outcome 'an exit file ends the watch'
+    Assert-Equal '0' $result.ExitText 'the exit file content is reported'
+    Assert-Equal "one`ntwo`nlast" $printed.ToString() 'log text is printed once, in order'
+    Assert-Equal '0 4 4' ($offsets -join ' ') 'each read continues at the offset the previous one returned'
+}
+
+Test-Case 'a guest watch that never sees an exit file ends at its deadline' {
+    $clock = New-WatchClock
+    $result = Watch-ReleaseVmGuestRun -TimeoutMinutes 1 -PollSeconds 15 -Now { $clock.Time } `
+        -Sleep { param($Seconds) $clock.Time = $clock.Time.AddSeconds($Seconds) } `
+        -Read { param($Offset) [pscustomobject]@{ Text = ''; Offset = 0; Done = $false; ExitText = $null } } `
+        -OnText { param($Text) }
+    Assert-Equal 'deadline' $result.Outcome 'an operator gate that is never answered must not hold the host'
+}
+
+Test-Case 'a guest watch gives up after consecutive failed reads, and a good read resets the count' {
+    $script:watchCalls = 0
+    $result = Watch-ReleaseVmGuestRun -MaxConsecutiveFailures 3 -Sleep { param($Seconds) } -OnText { param($Text) } `
+        -Read {
+            param($Offset)
+            $script:watchCalls++
+            if ($script:watchCalls -eq 3) {
+                return [pscustomobject]@{ Text = ''; Offset = 0; Done = $false; ExitText = $null }
+            }
+            throw 'guest gone'
+        }
+    Assert-Equal 'unreachable' $result.Outcome 'a guest that stops answering ends the watch'
+    Assert-Equal 6 $script:watchCalls 'two failures, one success, then three failures'
+    Assert-Match 'guest gone' $result.Detail 'the last failure is named'
+}
+
+Test-Case 'a progress read with no credential is refused instead of prompting' {
+    $refused = $false
+    try {
+        Read-ReleaseVmGuestProgress -VMName 'ExoSnap-None' -Credential $null -ExitFile 'C:\x' | Out-Null
+    }
+    catch { $refused = $_.Exception.Message -match 'Credential' }
+    Assert-True $refused 'a null credential reaching Invoke-Command opens a logon dialog on the host'
+}
+
+Test-Case 'the watch script refuses a missing argument without asking for it' {
+    $run = Invoke-Script -Path (Join-Path $script:VmRoot 'Watch-ReleaseVmRun.ps1') -Arguments @('-VMName', 'x')
+    Assert-Equal 1 $run.ExitCode 'a missing exit file is a usage failure'
+    Assert-Match '-ExitFile is required' $run.Output 'the refusal names the argument'
+    $source = Get-Content -LiteralPath (Join-Path $script:VmRoot 'Watch-ReleaseVmRun.ps1') -Raw
+    Assert-NoMatch '\[Parameter\(Mandatory' $source 'a mandatory parameter prompts when a host is interactive'
+}
 Write-Host ''
 Write-Host "  $($script:Passed) passed, $($script:Failed) failed."
 if ($script:Failed -gt 0) { exit 1 }

tools/vm/ReleaseVm.psm1

@@ -2783,6 +2783,140 @@ function Invoke-ReleaseVmCommand {
     return $result
 }
 
+function Read-ReleaseVmGuestProgress {
+    <#
+    .SYNOPSIS
+        One bounded read of a running guest campaign: new log lines and whether it finished.
+    .DESCRIPTION
+        The log is read from a byte offset and only complete lines come back, so a line
+        the campaign is still writing arrives whole on a later read. Once the exit file
+        exists the remainder is returned as is. The log is expected to be UTF-8; a
+        rewritten or truncated log restarts at its beginning.
+
+        The read runs as a job with its own timeout because a PowerShell Direct call
+        into a guest that stopped answering has no transport timeout. A timed-out job
+        is abandoned rather than stopped, since stopping it can block just as long; it
+        ends with the calling process.
+    #>
+    param(
+        [Parameter(Mandatory)] [string] $VMName,
+        [Parameter(Mandatory)] [System.Management.Automation.PSCredential] $Credential,
+        [Parameter(Mandatory)] [string] $ExitFile,
+        [string] $LogFile,
+        [long] $Offset = 0,
+        [ValidateRange(5, 600)] [int] $TimeoutSeconds = 60,
+        [ValidateRange(1024, 16777216)] [int] $MaxBytes = 262144
+    )
+    # The guest side runs in the guest's Windows PowerShell 5.1.
+    $job = Invoke-Command -VMName $VMName -Credential $Credential -AsJob -ScriptBlock {
+        param($ExitPath, $LogPath, $From, $Limit)
+        $done = Test-Path -LiteralPath $ExitPath
+        $exitText = $null
+        if ($done) { $exitText = "$(Get-Content -LiteralPath $ExitPath -Raw)".Trim() }
+
+        $text = ''
+        $next = $From
+        if ($LogPath -and (Test-Path -LiteralPath $LogPath)) {
+            $stream = [IO.File]::Open($LogPath, [IO.FileMode]::Open, [IO.FileAccess]::Read,
+                [IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete)
+            try {
+                if ($stream.Length -lt $From) { $From = 0 }
+                $count = [int][Math]::Min($stream.Length - $From, $Limit)
+                $bytes = New-Object byte[] $count
+                [void] $stream.Seek($From, [IO.SeekOrigin]::Begin)
+                $read = 0
+                while ($read -lt $count) {
+                    $n = $stream.Read($bytes, $read, $count - $read)
+                    if ($n -le 0) { break }
+                    $read += $n
+                }
+                $take = $read
+                if (-not $done -and $read -gt 0 -and $read -lt $Limit) {
+                    $take = [Array]::LastIndexOf($bytes, [byte] 10, $read - 1) + 1
+                }
+                $skip = 0
+                if ($From -eq 0 -and $take -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) {
+                    $skip = 3
+                }
+                $text = [Text.Encoding]::UTF8.GetString($bytes, $skip, $take - $skip)
+                $next = $From + $take
+            }
+            finally {
+                $stream.Dispose()
+            }
+        }
+        [pscustomobject]@{ Text = $text; Offset = $next; Done = $done; ExitText = $exitText }
+    } -ArgumentList $ExitFile, $LogFile, $Offset, $MaxBytes
+
+    if (-not (Wait-Job -Job $job -Timeout $TimeoutSeconds)) {
+        throw "'$VMName' did not answer a progress read within $TimeoutSeconds second(s)"
+    }
+    try {
+        $result = Receive-Job -Job $job -ErrorAction Stop
+    }
+    finally {
+        Remove-Job -Job $job -Force
+    }
+    if ($null -eq $result) { throw "'$VMName' returned no progress" }
+    return [pscustomobject]@{
+        Text = [string] $result.Text; Offset = [long] $result.Offset
+        Done = [bool] $result.Done; ExitText = $result.ExitText
+    }
+}
+
+function Watch-ReleaseVmGuestRun {
+    <#
+    .SYNOPSIS
+        Follows a guest campaign until it finishes, a deadline passes or the guest stops answering.
+    .DESCRIPTION
+        Every way out is bounded. Each read carries its own timeout, a run of failed
+        reads ends the watch as unreachable, and the deadline ends it whatever the guest
+        does. A campaign waiting on an operator shows as a quiet log, never as a host
+        that hangs. The watch only reads: it neither prompts nor changes the guest.
+
+        -Read receives the next offset and returns an object with Text, Offset, Done and
+        ExitText, as Read-ReleaseVmGuestProgress does. New text is passed to -OnText.
+        The result's Outcome is 'finished', 'deadline' or 'unreachable'.
+    #>
+    param(
+        [Parameter(Mandatory)] [scriptblock] $Read,
+        [Parameter(Mandatory)] [scriptblock] $OnText,
+        [ValidateRange(1, 1440)] [int] $TimeoutMinutes = 90,
+        [ValidateRange(1, 600)] [int] $PollSeconds = 15,
+        [ValidateRange(1, 100)] [int] $MaxConsecutiveFailures = 4,
+        [scriptblock] $Now = { [DateTime]::UtcNow },
+        [scriptblock] $Sleep = { param($Seconds) Start-Sleep -Seconds $Seconds }
+    )
+    $deadline = (& $Now).AddMinutes($TimeoutMinutes)
+    $offset = [long] 0
+    $failures = 0
+    while ((& $Now) -lt $deadline) {
+        try {
+            $progress = & $Read $offset
+            $failures = 0
+            if ($progress.Text) { & $OnText $progress.Text }
+            $offset = [long] $progress.Offset
+            if ($progress.Done) {
+                return [pscustomobject]@{ Outcome = 'finished'; ExitText = $progress.ExitText; Detail = '' }
+            }
+        }
+        catch {
+            $failures++
+            if ($failures -ge $MaxConsecutiveFailures) {
+                return [pscustomobject]@{
+                    Outcome = 'unreachable'; ExitText = $null
+                    Detail = "$failures consecutive reads failed; last: $($_.Exception.Message)"
+                }
+            }
+        }
+        & $Sleep $PollSeconds
+    }
+    return [pscustomobject]@{
+        Outcome = 'deadline'; ExitText = $null
+        Detail = "no exit file within $TimeoutMinutes minute(s); the campaign may still be running"
+    }
+}
+
 function Copy-ReleaseVmDirectoryBack {
     <#
     .SYNOPSIS
@@ -2914,4 +3048,6 @@ Export-ModuleMember -Function @(
     'Assert-ReleaseVmSealed'
     'Invoke-ReleaseVmProvisioning'
     'Invoke-ReleaseVmCommand'
+    'Read-ReleaseVmGuestProgress'
+    'Watch-ReleaseVmGuestRun'
 )

tools/vm/Watch-ReleaseVmRun.ps1

@@ -0,0 +1,74 @@
+#Requires -Version 7.0
+<#
+.SYNOPSIS
+    Follows a campaign running in a release-verification guest, read-only and bounded.
+    Runs on the HOST.
+
+.DESCRIPTION
+    Prints the guest campaign's log as it grows and returns once the guest writes its
+    exit file. The watch never waits without a limit: each read over PowerShell Direct
+    has its own timeout, repeated failed reads end it, and -TimeoutMinutes ends it
+    regardless. It never prompts. A missing argument is an error, not a question, so
+    it is safe to run unattended or from a background monitor.
+
+    Exit codes: 0 the exit file appeared (its content is printed, not propagated),
+    2 the deadline passed, 3 the guest stopped answering, 1 anything else.
+
+.PARAMETER VMName
+    The running guest to read from.
+
+.PARAMETER ExitFile
+    Guest path the campaign writes when it is done.
+
+.PARAMETER LogFile
+    Guest path of a UTF-8 log to follow. Optional.
+
+.EXAMPLE
+    pwsh -NoProfile -NonInteractive -File tools/vm/Watch-ReleaseVmRun.ps1 -VMName ExoSnap-Run-smoke-001 `
+        -ExitFile $guestExitFile -LogFile $guestLogFile -TimeoutMinutes 60
+#>
+[CmdletBinding()]
+param(
+    # Not Mandatory: PowerShell asks for a missing mandatory argument, and a question
+    # nobody sees stalls an unattended watch.
+    [string] $VMName,
+    [string] $ExitFile,
+    [string] $LogFile,
+    [ValidateRange(1, 1440)] [int] $TimeoutMinutes = 90,
+    [ValidateRange(1, 600)] [int] $PollSeconds = 15,
+    [ValidateRange(1, 100)] [int] $MaxConsecutiveFailures = 4
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$code = 1
+try {
+    if (-not $VMName) { throw '-VMName is required' }
+    if (-not $ExitFile) { throw '-ExitFile is required' }
+
+    Import-Module (Join-Path $PSScriptRoot 'ReleaseVm.psm1') -Force -DisableNameChecking
+    $credential = New-ReleaseVmCredential
+    $read = {
+        param($Offset)
+        Read-ReleaseVmGuestProgress -VMName $VMName -Credential $credential -ExitFile $ExitFile `
+            -LogFile $LogFile -Offset $Offset
+    }.GetNewClosure()
+    $print = { param($Text) [Console]::Out.Write($Text); [Console]::Out.Flush() }
+
+    $result = Watch-ReleaseVmGuestRun -Read $read -OnText $print -TimeoutMinutes $TimeoutMinutes `
+        -PollSeconds $PollSeconds -MaxConsecutiveFailures $MaxConsecutiveFailures
+    switch ($result.Outcome) {
+        'finished' { [Console]::Out.WriteLine("campaign finished; exit file: $($result.ExitText)"); $code = 0 }
+        'deadline' { [Console]::Out.WriteLine("watch ended: $($result.Detail)"); $code = 2 }
+        'unreachable' { [Console]::Out.WriteLine("watch ended, '$VMName' unreachable: $($result.Detail)"); $code = 3 }
+    }
+}
+catch {
+    [Console]::Error.WriteLine("watch failed: $($_.Exception.Message)")
+    $code = 1
+}
+[Console]::Out.Flush()
+# A normal exit disposes the runspaces of abandoned PowerShell Direct reads, and that
+# blocks for as long as the guest does not answer.
+[Environment]::Exit($code)