diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..fe37176
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,69 @@
+name: CI
+
+on:
+ push:
+ pull_request:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ windows:
+ runs-on: windows-2025
+ timeout-minutes: 20
+ defaults:
+ run:
+ shell: pwsh
+ env:
+ MSGUIDE_GUIDANCE_PROVIDER: demo
+ MSGUIDE_SESSION_SCREEN_CONTEXT: "0"
+ MSGUIDE_SESSION_AUTOMATION: "0"
+ MSGUIDE_WHISPER_SYNTHETIC_TEST: "0"
+ MSGUIDE_SPEECH_SYNTHETIC_TEST: "0"
+ MSGUIDE_DIAGNOSTIC_LOG: ""
+ MSGUIDE_DESKTOP_LOG: ""
+ PYTHONUTF8: "1"
+ DOTNET_NOLOGO: "true"
+ DOTNET_CLI_TELEMETRY_OPTOUT: "1"
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ clean: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.11.9"
+ architecture: x64
+ pip-version: "26.2.1"
+ cache: pip
+ cache-dependency-path: |
+ requirements.lock.txt
+ requirements-dev.lock.txt
+ - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+ with:
+ global-json-file: global.json
+ cache: true
+ cache-dependency-path: |
+ desktop\packages.lock.json
+ desktop\CaptureProbe\packages.lock.json
+ - name: Install hash-locked Python dependencies
+ run: |
+ python -m venv --without-pip .venv &&
+ python -m pip --python .venv install --require-hashes --only-binary=:all: -r requirements-dev.lock.txt &&
+ python -m pip --python .venv check
+ - name: Backend and dependency-contract regressions
+ run: .\.venv\Scripts\python -m pytest -q -p no:cacheprovider
+ - name: Restore locked .NET dependencies
+ run: dotnet restore .\desktop\CaptureProbe\MSGuide.CaptureProbe.csproj --locked-mode --verbosity minimal
+ - name: Build desktop and diagnostic probe
+ run: dotnet build .\desktop\CaptureProbe\MSGuide.CaptureProbe.csproj --configuration Release --no-restore --nologo --verbosity minimal
+ - name: Desktop synthetic self-tests
+ run: |
+ $report = Join-Path $env:RUNNER_TEMP 'msguide-self-test.json'
+ $process = Start-Process -FilePath .\desktop\bin\Release\net10.0-windows10.0.19041.0\MSGuide.Desktop.exe `
+ -ArgumentList '--self-test', '--test-results', "`"$report`"" -Wait -PassThru
+ if ($process.ExitCode -ne 0) { throw "Desktop self-test failed: $($process.ExitCode)" }
+ $result = Get-Content -LiteralPath $report -Raw | ConvertFrom-Json
+ if ($result.passed -ne $true) { throw 'Desktop self-test report did not pass.' }
+ Get-Content -LiteralPath $report
diff --git a/.gitignore b/.gitignore
index 6384c6f..00ed9cd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,8 @@ __pycache__/
htmlcov/
desktop/bin/
desktop/obj/
+desktop/CaptureProbe/bin/
+desktop/CaptureProbe/obj/
.env
.env.*
!.env.example
diff --git a/Directory.Build.props b/Directory.Build.props
new file mode 100644
index 0000000..f860221
--- /dev/null
+++ b/Directory.Build.props
@@ -0,0 +1,5 @@
+
+
+ true
+
+
diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md
index cf4eb96..800ad51 100644
--- a/IMPLEMENTATION.md
+++ b/IMPLEMENTATION.md
@@ -1,25 +1,27 @@
# Implementation status
-Updated September 14, 2026. This is a **local, single-user, guide-only desktop MVP**, not an enterprise service. Start with [README.md](README.md); evidence and remaining checks are in [docs/VALIDATION.md](docs/VALIDATION.md).
+Updated September 18, 2026. This is a **local, single-user desktop MVP with non-executing Guide mode and bounded, locally grounded plan execution**, not an enterprise service or unrestricted agent. Start with [README.md](README.md); historical evidence and remaining live checks are in [docs/VALIDATION.md](docs/VALIDATION.md).
## Implemented path
1. [scripts/Start-MSGuide.ps1](scripts/Start-MSGuide.ps1) builds WPF, starts the loopback backend on port 8765 by default, and gives both children an ephemeral local bearer token. Only the backend inherits the model API key. Server lifetime is tied to desktop/launcher cleanup; no token file is created.
-2. [desktop/MainWindow.xaml.cs](desktop/MainWindow.xaml.cs) manages the hotkey, window selection, editable prompt, capture review, consent, explicit Send, fresh Check, and cancellation.
-3. [desktop/CaptureService.cs](desktop/CaptureService.cs) captures the selected HWND with `PrintWindow` and collects bounded UI Automation names/boxes. It provides a real local preview, not OCR or redaction. PNGs are bounded to 1600 pixels per side and 2,000,000 bytes.
+2. [desktop/CompanionWindow.cs](desktop/CompanionWindow.cs) provides the normal Clicky-style shell: a click-through Windows-logo buddy and response bubble follow the cursor at 60 FPS, while the hotkey opens a compact interactive prompt with monitor-edge clamping. [desktop/MainWindow.xaml.cs](desktop/MainWindow.xaml.cs) remains the orchestration and expanded review/approval surface.
+3. [desktop/CaptureService.cs](desktop/CaptureService.cs) uses selected-HWND Windows Graphics Capture, without desktop/PrintWindow fallback, and cached, bounded UIA evidence prioritized by actionability. Non-action context cropping does not invalidate a complete controls scan; actual traversal failures remain blocked. Logical control identity is separate from the exact reviewed-state target token. It provides a local preview, not OCR/redaction; PNGs are bounded to 1280 pixels on their longest side and 2,000,000 bytes.
4. [desktop/ApiClient.cs](desktop/ApiClient.cs) calls health, sessions, and guidance only. It rejects non-loopback destinations, disables proxies/redirects, and does not call legacy assist/action routes.
5. [src/main.py](src/main.py) checks the local boundary, session, consent, freshness, body limits, provider output, and target correspondence. It defines **10 application routes**, excluding FastAPI's four generated documentation/schema routes; see [docs/API.md](docs/API.md).
-6. [src/guidance.py](src/guidance.py) implements the deterministic built-in demo. [src/model_provider.py](src/model_provider.py) implements the opt-in OpenAI-compatible transport; neither provider can execute tools. Model setup is [separate from local security mode](docs/MODEL_SETUP.md).
-7. [desktop/OverlayWindow.cs](desktop/OverlayWindow.cs) draws a nonactivating target outline. [desktop/SpeechService.cs](desktop/SpeechService.cs) supplies installed Windows speech recognition/playback. The user clicks; Check begins another review cycle.
+6. [src/guidance.py](src/guidance.py) implements the deterministic built-in demo. [src/model_provider.py](src/model_provider.py) and [src/copilot_provider.py](src/copilot_provider.py) select only reviewed targets; neither provider executes desktop tools. Model setup is [separate from local security mode](docs/MODEL_SETUP.md).
+7. [desktop/ScreenTaskSession.cs](desktop/ScreenTaskSession.cs) validates/retains up to 32 steps per plan and executes continuously without eight-action or two-minute checkpoints. Every action is observed; suitable post-action evidence is reused for the next local binding. After progress, plan-limit/observation boundaries trigger a fresh approved capture and next model plan only if the same identified resource remains completely inspectable. Empty plans do not spin. [desktop/DesktopAction.cs](desktop/DesktopAction.cs) still reacquires/checks each exact target, value, capability and window before invoking. Deferred intents require unique fresh complete matches and deferred writes require empty fields. Resource changes, missing grounding and permission/info boundaries stop explicitly. Per-operation deadlines and the 10,000-decision protocol ceiling remain; unknown and cancelled queues cannot resume.
+8. The compact interactive prompt and Details share mode, continuation/reply and Stop handlers. Generic actions bring only the approved window forward from the companion, visibly place the Windows marker/outline on the target, then revalidate before invocation; unrelated foreground changes stop rather than cause background input. The cursor buddy/overlay remain click-through and non-activating. Speech transcript invalidation is separate from hardware shutdown acknowledgement; stopping/unknown input gates new recordings and input changes, and late closure never revives cancelled text. Whisper model-load, processor and inference timings are separate; no speculative factory cache was added.
## Implemented safeguards, not enterprise guarantees
- The API accepts loopback clients/local Host values and rejects browser Origin headers. `/v1` and `/admin/` require the configured bearer token before body parsing. All authenticated calls share the local principal; no employee identity or cross-user authorization is established.
- Requests are bounded to 3,000,000 bytes. Observations expire after 60 seconds, with at most five seconds of future clock skew. The desktop additionally checks window identity/bounds and response echo IDs.
-- [src/models.py](src/models.py) validates normalized target boxes and confidence. Targets must match reviewed UIA evidence; model output selects an existing element index, never arbitrary coordinates. Model completion claims become clarification with a verification warning.
+- [src/models.py](src/models.py) validates complete plan shapes and boundaries before any first action. Observed references become server-derived logical IDs; future intents are bounded exact descriptions, not invented IDs/selectors. Generic completion remains a model suggestion with `review_required` UI, never independently verified goal success.
+- Execution remains desktop-owned: supported UIA invocation, toggling, selection, expansion, bounded full-field replacement and small scrolling only. Password/read-only/value/identity/window checks remain live. No arbitrary typing, dragging, coordinate input, shell or filesystem tool execution is added. Supported Edge/Chrome windows use a locally checked browser-chrome address hash; other document trees without proven file/site identity require handoff; partial approved evidence can still support Guide descriptions.
- [src/images.py](src/images.py) uses Pillow to decode/verify bounded PNGs and re-encode pixels without metadata. Valid PNGs are accepted even in demo mode, but deterministic guidance ignores them. Sharing pixels is unnecessary for that demo.
-- Capture/upload is manual. Prompt or approval changes, cancellation, and supersession invalidate pending work. Native capture runs on a worker with a 30-second waiting budget; it cannot be safely force-aborted. One stuck worker can block later captures until restart.
-- Evidence is not deliberately persisted. Clearing references/arrays does not guarantee erasure of all managed/native copies. Remote retention is the configured provider's policy, not controlled here.
+- Manual developer sessions require capture review and approval. A `-Copilot` launch uses its process-lifetime screen-context grant for foreground capture. Prompt or consent changes, cancellation, and supersession invalidate pending work. Native capture runs on a worker with a 30-second waiting budget; it cannot be safely force-aborted. One stuck worker can block later captures until restart.
+- Evidence is not deliberately persisted. Bounded rotating diagnostics under `%LOCALAPPDATA%\MSGuide\logs` contain operational metadata only, never screenshots, transcripts, tokens, UI text, prompts, or model output. Clearing references/arrays does not guarantee erasure of all managed/native copies. Remote retention is the configured provider's policy, not controlled here.
## Backend-only sample features
@@ -31,6 +33,11 @@ Sessions, previews, jobs, and optional audit events are bounded process-local st
## Evidence and gaps
-The parent validation run reports **149 pytest passes**, a clean dependency check, successful .NET build, and successful desktop self-test. Provider tests use synthetic evidence and mocked transport, not an approved live model. The real harness failed when `demo.Activate()` returned false after visible layout/rendering; foreground restrictions are only an unconfirmed explanation.
+The new offline suites exercise both fake providers/API, whole-plan rejection,
+three and 17 steps with one model call, batch continuation, scope/target drift,
+stable effects after label/position changes, compact handlers, and fake-input
+stop timeout/late acknowledgement. These are not live-model/app/device acceptance
+or native Whisper performance measurements. Older build/test counts and camera/
+voice observations in historical runbooks predate these changes.
-Do not infer capture, target placement, microphone, mixed-DPI behavior, or end-to-end desktop success from build/unit checks. See [docs/VALIDATION.md](docs/VALIDATION.md) and the unchecked milestones in [PLAN.md](PLAN.md). No OCR, enterprise access, internal-data pilot, full dependency lock, or production deployment is complete.
+Do not infer capture, target placement, microphone, mixed-DPI behavior, or end-to-end desktop success from build/unit checks. See [docs/VALIDATION.md](docs/VALIDATION.md) and [PLAN.md](PLAN.md). Python/NuGet locks and CI do not constitute live acceptance. OCR, enterprise access, an internal-data pilot and production deployment remain out of scope.
diff --git a/README.md b/README.md
index 7fc8cc1..ffeb8ae 100644
--- a/README.md
+++ b/README.md
@@ -1,65 +1,239 @@
# MSGuide desktop MVP
-A Windows WPF companion with **Guide me** and **Do it for me** modes. The approved local task invokes two controls in its synthetic demo. An experimental Notepad adapter can guide or insert an approved draft into a selected empty editor; external writes are disabled by default pending native acceptance. This is not general screen control. The default provider is deterministic, not AI; an explicitly configured [optional model provider](docs/MODEL_SETUP.md) can process approved synthetic/public evidence.
-
-**Status, September 15, 2026:** desktop build, safety/component tests, strict native demo Control and the four-state capture/API test pass. A DPI-triggered overlay activation bug was fixed; full integration passed three post-fix runs, including two with stronger zero-activation and hide/re-show checks. Notepad policy tests use a fake editor; real Notepad acceptance still needs user interaction and writes remain gated. Historical intermittent blank captures, full manual UX, mixed-DPI visual quality, microphone operation and live model quality remain unresolved or unverified. See [Notepad acceptance](docs/NOTEPAD_TASK.md), [dual-mode scope](docs/DUAL_MODE.md) and [validation evidence](docs/VALIDATION.md).
+A Windows companion for asking by text or voice and getting help with Microsoft
+tools. **Guide me** never executes actions, including in a `-Copilot` session.
+**Fix it for me** uses only supported, freshly revalidated actions under the
+launch-time automation grant; manual developer sessions require action approval.
+Current workflows include local Teams camera
+diagnosis and approved recovery, local voice transcription, and reviewed screen
+context for guidance. This is not unrestricted desktop control. Synthetic
+developer workflows are hidden unless explicitly enabled. The
+[provider configuration](docs/MODEL_SETUP.md) determines whether approved
+public/synthetic screen context uses Copilot or deterministic sample guidance.
+
+**Current test and presentation:** see the
+[Teams camera and voice walkthrough](docs/teams-camera-demo/README.md) for setup,
+live recovery, permission approvals, voice input, repeat-recording behavior, and
+regression commands. Live camera recovery and local voice input were exercised
+on Windows 11 ARM64 with English Teams on September 16, 2026. This is not a claim
+of arbitrary-app, all-device, or multilingual automation. Historical capture
+limits and separate Notepad acceptance remain documented in
+[validation](docs/VALIDATION.md) and [Notepad acceptance](docs/NOTEPAD_TASK.md).
## Install and start
Use an unlocked Windows 10/11 interactive desktop with:
- **PowerShell 7** (`pwsh`), not Windows PowerShell 5.1. The launcher uses modern .NET `ProcessStartInfo.ArgumentList` and `Environment` APIs.
-- **.NET 10 SDK** for the WPF build.
-- **Python 3.11**, available as `python` for initial setup.
+- **.NET SDK 10.0.400** (or a compatible patch allowed by `global.json`) for WPF.
+- **Windows x64 CPython 3.11.9**, available as `python` for the checked-in Python locks. ARM64 Windows can use x64 Python; a native ARM64 Python lock is not claimed.
From the project root in PowerShell 7:
```powershell
python -m venv venv
-.\venv\Scripts\python -m pip install -r requirements-dev.txt
+.\venv\Scripts\python -m pip install --require-hashes --only-binary=:all: -r requirements-dev.lock.txt
+dotnet restore .\desktop\MSGuide.Desktop.csproj --locked-mode
.\scripts\Start-MSGuide.ps1
```
-No virtual-environment activation is required. The launcher expects the environment at the root's venv directory. [requirements-dev.txt](requirements-dev.txt) includes [requirements.txt](requirements.txt); these pin direct requirements, not a complete transitive dependency lock.
-
-[scripts/Start-MSGuide.ps1](scripts/Start-MSGuide.ps1) builds the desktop, starts a single-worker loopback API on **port 8765**, checks readiness, and opens the companion. It creates an ephemeral local token, writes no token to disk, and removes `MSGUIDE_MODEL_API_KEY` from the desktop child's environment. On desktop exit or launcher cleanup, it stops its owned server. Keep the launcher running for the desktop session.
+No virtual-environment activation is required. The launcher expects the root's venv directory. The portable `requirements-dev.lock.txt` contains the tested runtime and test dependency closure; use `requirements.lock.txt` instead for runtime-only installation, retaining `--require-hashes --only-binary=:all:`. These files pin versions and wheel hashes without embedding workstation-specific package-mirror URLs or unrelated environment packages. Ordinary installation does not require experimental lockfile support or pip 26.2.1.
+
+Keep direct dependency intent in `requirements.txt` / `requirements-dev.txt`. Regenerate locks intentionally after manifest changes using `scripts\lock_python_dependencies.py` with pip 26.2.1; the helper converts native pip lock output using the standard library. Do not weaken TLS validation to resolve package-source problems. NuGet lock files cover the desktop and capture probe, and launcher builds use locked restore.
+
+[scripts/Start-MSGuide.ps1](scripts/Start-MSGuide.ps1) builds the desktop, starts a single-worker loopback API on **port 8765**, checks readiness, and opens the companion. Readiness has a 60-second budget, leaving headroom beyond the provider's 30-second startup deadline; failures identify the readiness stage and diagnostic log. Press `Ctrl+Alt+M`, or click the stationary logo, to open the compact prompt. Camera recovery, permission approvals, voice, Settings, manual screen review, and task progress all use this one scrollable view; there is no separate Details step.
+
+An approved generic observation requests a structured **plan segment**: up to **32 ordered steps**, ending at suggested completion or an explicit resource, information, permission, observation, unsupported-operation, or plan-limit boundary. The entire plan is validated before its first action. In **Fix it for me**, one model call can drive multiple steps using fresh local UIA checks, not one model call per click. Execution continues without an eight-action or two-minute pause and verifies every action. After a segment makes safe progress, `plan_limit` and `observation` boundaries automatically refresh the same approved resource and request the next segment. The 32-step response limit is not an execution checkpoint. A changed resource, missing/changed target, incomplete evidence, required input/permission, unsupported operation, cancellation, no progress, or unknown outcome still stops the run. An empty plan never triggers repeated automatic planning. **Review & continue** is for resumable interruptions, not routine step counts; suggested completion still requires review. The existing 10,000-decision protocol ceiling remains a safeguard against pathological runs.
+
+A task retains its original request, plan/cursor, task/step IDs, last 16 action outcomes, and bounded clarification text in memory. Repeated controls are allowed on progressed states; every step is uniquely rebound locally and its native target/state is checked again. Deferred writes require an empty writable non-password field; observed writes require the unchanged reviewed value digest. Unknown outcomes and cancelled queued work cannot resume. Generic goal completion remains **not independently verified**, even after observed control effects and a model completion suggestion.
+
+The normal hotkey remains **Ctrl+Alt+M**. Other keys require an explicit
+`MSGUIDE_HOTKEY` override; they are not a new default. Close an older MSGuide copy
+before relaunching an updated build. If registration conflicts, the new copy
+keeps the compact prompt visible with a taskbar entry instead of hiding with no usable hotkey.
+
+**Settings > Companion position** contains the **Follow pointer**
+option. Turn it off to pin the companion in place. Drag the pinned logo or the
+**Move** control in Settings to reposition it; focus **Move** and use arrow
+keys for keyboard movement (Shift+arrow moves one pixel). The choice and position
+are saved locally on this PC, with off-screen positions brought onto an available
+monitor. This is independent of **Guide me / Fix it for me**. An approved action
+can still show its click-through target marker, then return to the pinned position.
+Only this display preference and coordinates are saved, not prompts or approvals.
+Click the stationary logo to open the same prompt as **Ctrl+Alt+M**; dragging
+repositions it without opening the prompt. The logo has no rectangular button
+frame. The following logo and action markers remain click-through.
+
+The compact prompt has a 44-DIP **microphone icon** beside Ask that becomes a stop icon
+while recording, with clear text/tooltips and screen-reader labels. **Append**
+explicitly adds to a draft; cancellation stays immediately available while
+processing or input closure is uncertain. The input meter, warnings, and
+**Settings > Microphone options** retain input selection/refresh and idle audio controls. It shares the same draft
+and local speech lifecycle as the rest of the app. Opening the prompt never starts recording;
+closing it cancels pending voice input. Transcription never submits the question.
+
+For **Fix it for me**, submitting a Teams camera request authorizes one
+freshly revalidated camera-on action for the selected meeting/prejoin. MSGuide
+inspects first, opens Camera settings if needed, and continues through local
+checks automatically; there are no routine Next buttons. Device-wide camera
+access, user-app access, Teams permission, and Teams restart still require
+separate approvals in the compact view. Permission-on and camera-on alone do not
+prove readiness. Stop, dismissal, edits, mode changes, and changing the selected
+Teams window revoke pending camera-on authority. Ambiguous windows, unsupported
+surfaces, policy blocks, and unknown outcomes require a handoff, not retries.
+**Guide me** still leaves changes to the user.
+Both modes are shown as direct, mutually exclusive choices with a visible
+selection mark. Click the mode you want; selecting the already-active mode does
+nothing. Choosing the other mode stops pending work and clears its approvals
+before switching. It never submits the draft or restarts a repair automatically.
+
+MSGuide identifies the meeting/prejoin by a fresh, complete read of its camera
+controls, rather than selecting Teams' home/chat window by title. A disabled
+camera can identify the meeting without authorizing any action. The invoked
+window is preferred only when its camera surface is verified; otherwise the
+unique matching meeting is selected. Multiple matching meetings or incomplete
+reads still require clarification. The compact prompt fits its content rather
+than holding a full-size workspace open, while retaining its position during
+status changes. Read-only **Check again** results stay in the
+same camera card. **Fix camera** starts a fresh, separately requested repair.
+The current action, permission scope, Stop, and any restart/error warning stay
+in the main view; secondary progress and controls are under **More options**.
+Verified completion shows **Resolved · camera ready** with **Done** as the main
+action and **Check again** secondary. Done clears the finished question and
+returns to the small companion without changing the camera or its permissions.
+Already-working results still say no change was needed; fixtures say Fixture
+complete, never real resolution.
+
+Generic Fix-mode actions are presented **in the foreground**: the target is
+outlined and the Windows-logo marker moves onto it before invocation. The app
+can return focus from its own companion to the approved window, but never steals
+focus from an unrelated application or falls back to background input. Switching
+away, cancellation, or a stale target prevents the action.
+
+The launcher creates an ephemeral local token, writes no token to disk, and removes model credentials from the desktop child's environment. On exit it stops its owned server. Bounded rotating diagnostics under `%LOCALAPPDATA%\MSGuide\logs` connect task/step IDs with capture, guidance, invocation, verification, and stop timings; they exclude prompts, labels, typed values, screenshots, and model prose.
- `-Port 8766`: choose a different free port (1024–65535); existing processes are never stopped to free a port.
- `-SkipBuild`: reuse an existing desktop binary; omit after source changes.
- `-IntegrationTest`: run the synthetic desktop harness instead of normal UI; forces the deterministic provider. See [how to run validation](docs/VALIDATION.md).
- `-CaptureTest`: test real demo-window capture and API guidance without requiring foreground activation. This does not test overlay interaction.
+- `-Copilot`: use the persistent GitHub Copilot SDK provider for the generic
+ snapshot-guidance workflow, pinned to GPT-6 Astra with low reasoning and
+ the default context tier for interactive latency. Launching with this explicit switch grants screen
+ context for that process lifetime: invoking MSGuide from an app and asking a
+ question captures and sends that foreground window automatically. The same
+ launch consent authorizes bounded planned semantic actions after fresh local grounding
+ only in **Fix it for me**. **Guide me** remains non-executing. The Teams camera journey keeps its local state,
+ target revalidation, approval, and invocation boundary.
+- `-CameraFixture`: run the camera recovery card against its clearly labelled
+ deterministic Teams-camera-off fixture. Choose **Guide me** or
+ **Fix it for me**; fixture completion never claims real camera recovery.
+- `-Shareable`: explicitly allow MSGuide and its guidance overlay to appear in
+ full-screen sharing. They are excluded from capture by default. Share the
+ **Screen**, not an individual Teams window.
+- `-DeveloperTools`: show the synthetic Build Center/Notepad workflows and
+ manual screen-context diagnostics. Hidden during normal use.
+
+The normal shell starts with a blank **Ask MSGuide** composer and **Guide me** /
+**Fix it for me** modes. Task-specific controls appear after a request; Settings
+holds voice, connection, and privacy options. When the requested task is not the camera workflow, **Use screen context**
+opens explicit capture/review/approval. An explicit `-Copilot` launch authorizes
+freshly grounded planned semantic actions (`invoke`, `toggle`, `select`, `expand`,
+`collapse`, `set_value`, or `scroll`) in Fix mode. `set_value`
+replaces an entire writable, non-password field with an explicit value of at
+most 1000 characters; its previous value fingerprint must still match. `scroll`
+moves one small UIA increment in an explicitly allowed direction. Unsupported
+editors, read-only/password fields, incomplete trees, and non-actionable surfaces
+require a handoff; there is no coordinate/keyboard fallback. Guide mode can still
+describe an approved partial tree/image without executing it.
## Try the built-in demo
+These older synthetic workflows require `-DeveloperTools`. For the primary live
+scenario, use the [camera and voice walkthrough](docs/teams-camera-demo/README.md).
+
For the new dual-mode task: **Open demo → choose Guide me / Do it for me → Prepare demo task → review and approve → Start approved task**. Guide mode waits for your clicks and **I did it · check**. Control mode invokes View logs and Open troubleshooting, then revokes authority. **Stop task** and **Take over manually** remain available. See [the complete dual-mode instructions](docs/DUAL_MODE.md).
-The separate snapshot-guidance workflow is:
+The easiest real Clicky-style test is:
+
+1. Start with `.\scripts\Start-MSGuide.ps1 -Copilot`. The full workspace stays hidden and a small Windows-logo buddy follows the pointer.
+2. Put Calculator or another supported desktop app in the foreground, move the pointer near the control you want help with, and press `Ctrl+Alt+M`.
+3. Ask a concrete question in the compact prompt, such as `Where do I clear this calculation?`, then press Enter. The prompt disappears, the buddy shows its thinking state, and the foreground app is captured under the launch-time screen-context grant.
+4. In **Guide me**, follow the descriptive plan yourself. Choose **Fix it for me** in the compact prompt to use the launch grant. A mode change revokes old queued authority. Fix mode reacquires each exact fresh UIA target, invokes once, and observes the result. Reopen the compact prompt to review/continue, answer a clarification, or stop the retained task. Suggested completion is not proof.
+
+For the deterministic developer harness, start with `.\scripts\Start-MSGuide.ps1 -DeveloperTools`, open the compact prompt, expand **Developer tools**, and choose **Start demo · capture locally**.
+2. Inspect the actual image and UI Automation text/boxes. Leave screenshot sharing off, check explicit consent, then click **Send approved snapshot**.
+3. Briefly switch to the demo to see the target. The manual developer harness retains its explicit action button; `-Copilot` sessions act automatically only in Fix mode. The
+ click-through outline includes the floating Windows-logo marker requested for
+ the desktop UI.
+4. Use **Check next step**, review/send the fresh capture, and repeat for **View logs → Open troubleshooting → Mark resolved**. The final demo screen says **Issue resolved**.
+5. Use **Pause / clear**, **Dismiss**, Escape, minimize, or Exit to cancel work and clear the current evidence/transcript. Already transmitted data cannot be recalled.
+
+Editing the prompt replaces the old task. Changing the selected window or moving/resizing it invalidates current evidence. Evidence still expires after 60 seconds: SDK/API/desktop guidance waits use the **remaining** lifetime, with 10/8/6 seconds reserved respectively. They do not extend evidence validity. Native actions are caller-bounded to eight seconds; a hung COM call cannot be interrupted safely and blocks new actions until it returns or MSGuide is restarted.
-1. Click **Open demo**, then invoke MSGuide with **Ctrl+Alt+M**. The hotkey is configurable through `MSGUIDE_HOTKEY`; use the taskbar if registration fails.
-2. Enter “Help me find the build error” **before capture**, and select **MSGuide Demo**.
-3. Click **Capture / review**. Inspect the actual image and UI Automation text/boxes, including full-size image inspection. Nothing has been uploaded.
-4. Leave screenshot sharing off for the default demo. Check explicit consent, then click **Send approved snapshot**. Consent alone does not send anything.
-5. Switch to the demo and click the indicated control yourself: **View logs → Open troubleshooting → Mark resolved**.
-6. After each click, use **Check next step** for a **fresh capture and review**, approve again, and send. It is not an automatic observation loop. The final demo screen says **Issue resolved**.
-7. Use **Pause / clear**, **Dismiss**, Escape, minimize, or Exit to cancel work and clear the current evidence/transcript. Already transmitted data cannot be recalled.
+The initial plan request and an explicitly requested replan can share an approved screenshot; steps inside the segment and post-action checks stay **local UIA-only**, reusing suitable post-action evidence for the next binding. Verification retries observations, not actions: up to six reads within five seconds. Semantic actions require their expected effect; only `invoke` can use a stable screen change, which is not causal or goal proof. Logical `controlId` survives label/position changes for verification; the separate `targetId` still binds the exact reviewed state before invocation.
-Editing the prompt, changing the selected window, or moving/resizing the captured window invalidates the snapshot. Snapshots expire after 60 seconds. In-window content changes are not all detected: always request a fresh Check after acting.
+Queued execution requires a stable selected-window resource scope. Supported English Microsoft Edge and Chrome windows can establish a page scope from one visible HTTP(S) address control in browser chrome, outside page documents, with one visible document surface. The address is hashed locally and rechecked before actions; ambiguous, unsupported, or changed page identity still stops execution. Other document trees without proven file/site identity require handoff. New windows are not selected automatically. SDK sessions remain isolated, and remaining plan/history is untrusted context, not cached execution authority.
+
+UIA inspection caches bounded per-node properties and prioritizes actionable
+controls plus scope markers in the 200-element export. Shortening decorative
+text/context alone no longer disables an otherwise complete control scan.
+Actual traversal/provider failures or too many action controls still fail
+closed. Camera diagnosis retains its stricter complete-context requirement.
## Voice
-**Start microphone / Stop microphone** is local click-to-toggle dictation, auto-stopping after 30 seconds—not hold-to-talk or a wake word. Review/edit recognized text before capture; it is never automatically sent. **Speak response** uses local playback. Recognition depends on installed Windows speech recognizers, language support, and a usable default microphone; playback needs an installed voice. Type instead if these are unavailable. **Stop speech** also cancels pending guidance.
+Dictation uses **local Whisper (`small.en`)**, not the legacy Windows dictation
+engine or the Copilot model. No Developer Mode, MSIX registration, cloud audio
+processing, or speech subscription is needed. Install the model once, only
+after approving its approximately 466 MiB download:
+
+```powershell
+.\scripts\Install-MSGuideSpeechModel.ps1 -AcceptDownload
+```
+
+The installer verifies the model's SHA-256 digest and stores it under
+`%LOCALAPPDATA%\MSGuide\models`, outside the repository. The launcher never
+downloads models automatically; a missing model fails visibly with no silent
+fallback to another recognizer.
+
+Open **Voice settings**, choose the intended **Microphone**, then return to the
+question and select the **microphone icon**.
+With an existing draft, it is labeled **Replace with voice** and replaces
+the old text only after new speech is recognized. Use **Append** to explicitly
+append. Failed or cancelled replacement recordings retain the old draft; typing
+while recording cancels pending speech so it cannot overwrite your edits.
+The input meter shows incoming audio. **Stop & transcribe** closes the microphone
+and transcribes the bounded recording locally; recording auto-stops after 30
+seconds. Audio stays in memory and is cleared after processing or cancellation.
+Transcription can take several seconds and has a 45-second deadline. During
+processing, **Cancel transcription** stops the operation. Review/edit the words,
+then select **Ask MSGuide** or press Enter. Speech never submits automatically.
+
+Use **Sound input settings** for hardware mute or input-volume problems; choosing
+a microphone in MSGuide does not change Windows' default input. **Speak response**
+uses Windows local playback and requires an installed voice. Pause, dismissal,
+and exit cancel dictation and discard pending speech callbacks.
+During shutdown, **MIC STOPPING** remains visible and recording/input changes stay
+disabled until hardware closure is acknowledged. After a three-second missing-ack
+deadline, **MIC STATUS UNKNOWN** remains latched; a late confirmed closure can
+clear the gate but cannot revive cancelled speech. A permanently stuck driver
+may require closing the app. Model-load/factory, processor-creation and inference
+timings are recorded separately without audio or transcript content. No factory
+cache was added, and native performance was not benchmarked by the fake tests.
## Privacy and limits
-- Manual selected-window capture only; no whole-desktop fallback, periodic capture, or global input injection.
+- Selected-window evidence only; no whole-desktop fallback or global input injection. Session-approved task runs use bounded, action-driven observations, not an always-on monitor. Semantic field replacement and small scrolling require supported UIA patterns; arbitrary typing, coordinate clicks, dragging, and shell commands are not supported.
- **Snapshot UI Automation names only, not pixel OCR. No pixel redaction or redaction editor.** Excluding password controls from UIA does not sanitize screenshot pixels or all accessible text. Discard sensitive captures. The separately approved Notepad task reads bounded editor text locally for verification.
-- Screenshot upload is opt-in per snapshot. Pillow validates PNGs and strips metadata before remote processing; it does not remove sensitive pixels. The default backend makes no remote model calls.
-- Capture data is held in memory and cleared/disposed on completion or cancellation, not deliberately saved by the application. This is not guaranteed forensic memory erasure or a promise about a remote provider's retention.
+- Manual screenshot upload is opt-in per snapshot; the explicit Copilot launch grant covers initial planning and explicitly reviewed replanning observations. Pillow validates PNGs and strips metadata before remote processing; it does not remove sensitive pixels. Routine planned steps do not upload another observation. Local camera verification remains separate and is not automatically uploaded.
+- Images/current observations are cleared on stop. The original task, bounded plan/cursor, last 16 action records, and clarification/checkpoint text remain in local memory until a new prompt, Pause/clear, dismissal, or exit. No durable task store is added. This is not forensic memory erasure or a promise about remote retention.
- Protected, elevated, GPU-rendered, minimized, blank, or unresponsive windows may fail capture. A stuck native capture may require restarting MSGuide.
- No enterprise authentication, permission-aware search, general external automation, production deployment, or approved internal-data pilot. Experimental Notepad control requires explicit opt-in; use synthetic/public data only.
+- The desktop is the real semantic-action executor. `/v1/actions/*` and `/v1/jobs/*` remain the separate 250 ms in-memory **mock simulator**, not a task queue or desktop execution/status service.
## Documentation
+- [docs/teams-camera-demo/README.md](docs/teams-camera-demo/README.md): live test and presentation runbook, expected results, voice behavior, and repeatable checks.
- [docs/NOTEPAD_TASK.md](docs/NOTEPAD_TASK.md): experimental external task, test evidence and acceptance checklist.
- [docs/DUAL_MODE.md](docs/DUAL_MODE.md): implemented Guide/Control fixture, authorization, test results and external-control limits.
- [SUMMARY.md](SUMMARY.md): current status at a glance.
diff --git a/ROADMAP.md b/ROADMAP.md
index cee1a47..ce111e4 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -1,6 +1,6 @@
# Remaining acceptance gates
-The product direction now includes **Guide me** and permission-scoped **Do it for me**. The local dual-mode fixture and an [experimental Notepad adapter](docs/NOTEPAD_TASK.md) are implemented. Notepad writes default off pending native acceptance. Next: pass user-assisted foreground/Stop/takeover tests and prove the selected blank-Notepad task in both modes. General desktop control and model-generated actions remain unimplemented.
+The product direction now includes **Guide me** and permission-scoped **Do it for me**. The local dual-mode fixture, an [experimental Notepad adapter](docs/NOTEPAD_TASK.md), and a first generic selected-window action slice are implemented. An explicit Copilot launch authorizes the generic slice to perform one stable-ID-reacquired UIA `invoke`, `toggle`, `select`, `expand`, or `collapse` per response without another prompt; the model selects reviewed evidence but has no execution tool. Next: pass user-assisted foreground/Stop/takeover tests and broaden only from measured app-specific evidence. Typing, scrolling, coordinate input, ambient observation, and general desktop control remain unimplemented.
The current implementation is summarized in [SUMMARY.md](SUMMARY.md); detailed progress is in [PLAN.md](PLAN.md). This roadmap is not a production commitment or a claim of completed UX validation.
@@ -24,7 +24,7 @@ Validate setup on a clean Windows machine and record exact resolved dependencies
Before any internal-data pilot, obtain approved identity/data handling, implement server-validated enterprise authentication and permission-aware retrieval, and test access isolation. None is supplied by `MSGUIDE_MODE=demo` or model credentials.
-Keep real actions disabled until downstream authorization, least privilege, exact preview/confirmation binding, idempotency, cancellation semantics, and audit requirements are implemented and reviewed. The current action API only simulates effects; the desktop does not use it.
+Keep enterprise/downstream actions disabled until authorization, least privilege, exact preview/confirmation binding, idempotency, cancellation semantics, and audit requirements are implemented and reviewed. The backend action API still only simulates effects and the desktop does not use it. The local UIA slice is a separate per-action boundary and is not evidence that enterprise action requirements are complete.
## 5. Consider a controlled pilot only after the gates pass
diff --git a/SUMMARY.md b/SUMMARY.md
index 2d4982f..b1e522f 100644
--- a/SUMMARY.md
+++ b/SUMMARY.md
@@ -1,17 +1,20 @@
# Current status
-September 14, 2026. **Desktop-first local MVP; end-to-end runtime validation blocked.**
+September 18, 2026. **Desktop-first local prototype. Synthetic checks are not
+live-app or live-model acceptance.** Use [README.md](README.md) for the current
+behavior and setup; dated evidence remains in the scenario runbooks.
| Area | Evidence / limit |
| --- | --- |
-| Windows WPF shell | Implemented; .NET 10 build and desktop self-test pass. Not proof of runtime UX. |
-| Capture → guidance | Initial 18-check real-window pass; latest runs fail on blank captured pixels. Capture reliability and full foreground/overlay loop remain unverified. |
+| Windows WPF shell | Cursor companion and compact prompt, with Guide/Fix modes. Guide mode never executes actions. |
+| Capture and control | Selected-window capture and cached, action-priority UIA evidence. Generic actions visibly mark the target and require foreground execution. Edge/Chrome page identity is checked through browser chrome; unsupported, ambiguous or uncertain states remain blocked. |
+| Task planning and progress | Approved observations produce ordered plan segments. Fix mode runs continuously without eight-action or two-minute checkpoints; every action is freshly grounded and observed locally. Plan-limit/observation boundaries refresh automatically after progress on the same approved resource. Genuine resource/input/permission changes and uncertain outcomes still stop. Generic model-suggested completion requires review. |
| Default guidance | Deterministic built-in MSGuide Demo workflow only; unit-tested, no remote calls. |
-| Optional model | Explicit OpenAI-compatible transport implemented and unit-tested with mocks. No verified live-model grounding. |
-| Voice | Local click-to-toggle dictation, 30-second auto-stop, optional playback; installed Windows speech support required. Manual verification pending. |
-| Backend | 10 application routes; 150 pytest tests passed; pip check clean. |
-| Desktop integration | Failed at `demo-activate`: activation returned false after rendering/visible layout. Root cause unconfirmed. |
-| Security / retrieval / actions | Local bearer boundary, bundled public samples, simulated actions only. No enterprise identity or search. |
+| Optional models | Explicit Copilot SDK and OpenAI-compatible providers. Consent, bounded deadlines and cancellation remain required; fake-provider regressions do not prove live grounding or latency. |
+| Voice | Local Whisper `small.en`, selected microphone, bounded in-memory recording and reviewed transcripts. Windows speech is used for optional playback, not production dictation. |
+| Backend and mock actions | Loopback bearer boundary and public sample retrieval. `/v1/actions/*` and `/v1/jobs/*` are simulations; the desktop is the real semantic-action executor. |
+| CI and dependencies | Windows CI installs hash-locked Python dependencies and restores locked NuGet packages before backend/desktop synthetic checks. Python locks target CPython 3.11 x64; `global.json` selects the .NET SDK. A checked-in workflow is not evidence of a hosted CI pass. |
+| Native acceptance | Camera/voice evidence from September 16 is scoped to its documented environment. Earlier capture/foreground failures and pending acceptance are preserved in the validation history. New behavior needs its own matching live evidence. |
| Pilot / deployment | Not complete, not approved, no production deployment. |
## Start here
diff --git a/desktop/ApiClient.cs b/desktop/ApiClient.cs
index 01fde8d..43ba025 100644
--- a/desktop/ApiClient.cs
+++ b/desktop/ApiClient.cs
@@ -15,10 +15,18 @@ public sealed class ApiClient : IDisposable
public ApiClient()
{
http = new HttpClient(new HttpClientHandler { AllowAutoRedirect = false, UseProxy = false, UseCookies = false })
- { BaseAddress = Safety.ApiUri(Environment.GetEnvironmentVariable("MSGUIDE_API_URL")), Timeout = TimeSpan.FromSeconds(45), MaxResponseContentBufferSize = 512 * 1024 };
+ { BaseAddress = Safety.ApiUri(Environment.GetEnvironmentVariable("MSGUIDE_API_URL")), Timeout = TimeSpan.FromSeconds(54), MaxResponseContentBufferSize = 512 * 1024 };
token = Environment.GetEnvironmentVariable("MSGUIDE_LOCAL_TOKEN");
}
+ internal ApiClient(HttpMessageHandler handler, string localToken)
+ {
+ http = new HttpClient(handler)
+ { BaseAddress = Safety.ApiUri("http://127.0.0.1:8000"), Timeout = TimeSpan.FromSeconds(54),
+ MaxResponseContentBufferSize = 512 * 1024 };
+ token = localToken;
+ }
+
private async Task Send(HttpRequestMessage request, bool authenticated, CancellationToken ct)
{
using (request)
@@ -32,29 +40,85 @@ private async Task Send(HttpRequestMessage request, bool authenticated, Ca
using var response = await http.SendAsync(request, ct);
if (!response.IsSuccessStatusCode)
{
- session = null;
- throw new InvalidOperationException($"Local service returned HTTP {(int)response.StatusCode}. Check the service/token; capture and approve again. No automatic retry.");
+ if ((int)response.StatusCode is 401 or 404 or 410) session = null;
+ DiagnosticLog.Record("http_error", new
+ {
+ endpoint = request.RequestUri?.ToString(),
+ status = (int)response.StatusCode,
+ errorCode = Header(response, "X-MSGuide-Error-Code"),
+ errorType = Header(response, "X-MSGuide-Error-Type"),
+ correlationId = Header(response, "X-MSGuide-Correlation-ID"),
+ });
+ throw new InvalidOperationException(ServiceErrorMessage(response));
}
return await response.Content.ReadFromJsonAsync(Json, ct)
?? throw new InvalidOperationException("The local service returned an empty response.");
}
}
+ private static string ServiceErrorMessage(HttpResponseMessage response)
+ {
+ string? code = Header(response, "X-MSGuide-Error-Code");
+ return code switch
+ {
+ "provider-invalid-result" => "Copilot could not match the request to an approved control. Try the request again on the current screen.",
+ "provider-invalid-context" => "MSGuide could not prepare the captured controls for Copilot. Capture the window again.",
+ "guidance-invalid-result" => "Copilot returned a target that no longer matches the captured window. Capture the window again.",
+ "provider-not_started" or "provider-startup" or "provider-runtime" =>
+ "The Copilot provider could not complete the request. Restart MSGuide and try again.",
+ "guidance-unexpected" =>
+ $"MSGuide hit an unexpected guidance error ({SafeErrorType(response)}).",
+ _ when (int)response.StatusCode == 504 => "Guidance exceeded the remaining evidence budget. No action was accepted; capture fresh evidence to continue.",
+ _ => $"Local service returned HTTP {(int)response.StatusCode}. Restart MSGuide and try again.",
+ };
+ }
+
+ private static string SafeErrorType(HttpResponseMessage response)
+ {
+ string? value = Header(response, "X-MSGuide-Error-Type");
+ return value is { Length: > 0 and <= 128 } && value.All(c => char.IsAsciiLetterOrDigit(c) || c == '_')
+ ? value
+ : "Unknown";
+ }
+
+ private static string? Header(HttpResponseMessage response, string name) =>
+ response.Headers.TryGetValues(name, out var values) ? values.SingleOrDefault() : null;
+
public Task Health(CancellationToken ct) => Send(new(HttpMethod.Get, "health"), false, ct);
- public async Task Guide(Observation observation, string prompt, CancellationToken ct)
+ public async Task Guide(
+ Observation observation, string prompt, CancellationToken ct, TaskProgress? task = null, bool planSegments = true)
{
- if (session is null || session.ExpiresAt <= DateTimeOffset.UtcNow.AddSeconds(5))
+ var budget = Safety.GuidanceBudget(observation.CapturedAt, DateTimeOffset.UtcNow);
+ if (!Safety.Fresh(observation.CapturedAt, DateTimeOffset.UtcNow) || budget <= TimeSpan.Zero)
+ throw new InvalidOperationException("Too little freshness remains for guidance. Capture new evidence; no action was accepted.");
+ using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct);
+ deadline.CancelAfter(budget);
+ var requestToken = deadline.Token;
+ try
+ {
+ if (session is null || session.ExpiresAt <= DateTimeOffset.UtcNow.AddSeconds(5))
+ {
+ var created = await Send(new(HttpMethod.Post, "v1/sessions"), true, requestToken);
+ requestToken.ThrowIfCancellationRequested();
+ if (string.IsNullOrWhiteSpace(created.SessionId) || created.ExpiresAt <= DateTimeOffset.UtcNow)
+ throw new InvalidOperationException("The service returned an invalid or expired session.");
+ session = created;
+ }
+ requestToken.ThrowIfCancellationRequested();
+ if (!Safety.Fresh(observation.CapturedAt, DateTimeOffset.UtcNow))
+ throw new InvalidOperationException("Snapshot expired. Capture and review again.");
+ var result = await Send(new(HttpMethod.Post, "v1/guidance")
+ { Content = JsonContent.Create(new GuidanceRequest(session.SessionId, prompt, true, observation, task, planSegments), options: Json) }, true, requestToken);
+ if (planSegments && result.Plan is null
+ || result.Plan is not null && !Safety.ValidPlan(result.Plan, observation))
+ throw new InvalidOperationException("The service returned a missing or invalid whole plan. No action was accepted.");
+ return result;
+ }
+ catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
- session = await Send(new(HttpMethod.Post, "v1/sessions"), true, ct);
- if (string.IsNullOrWhiteSpace(session.SessionId) || session.ExpiresAt <= DateTimeOffset.UtcNow)
- throw new InvalidOperationException("The service returned an invalid or expired session.");
+ throw new InvalidOperationException("Guidance timed out before the evidence expired. No action was accepted; review the retained task and capture fresh evidence.");
}
- ct.ThrowIfCancellationRequested();
- if (!Safety.Fresh(observation.CapturedAt, DateTimeOffset.UtcNow))
- throw new InvalidOperationException("Snapshot expired. Capture and review again.");
- return await Send(new(HttpMethod.Post, "v1/guidance")
- { Content = JsonContent.Create(new GuidanceRequest(session.SessionId, prompt, true, observation), options: Json) }, true, ct);
}
public void Dispose() { session = null; http.Dispose(); }
diff --git a/desktop/App.xaml b/desktop/App.xaml
index a6ba2e1..f3809a8 100644
--- a/desktop/App.xaml
+++ b/desktop/App.xaml
@@ -1,16 +1,483 @@
-
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
\ No newline at end of file
+
diff --git a/desktop/App.xaml.cs b/desktop/App.xaml.cs
index 942e05a..e60f539 100644
--- a/desktop/App.xaml.cs
+++ b/desktop/App.xaml.cs
@@ -9,6 +9,7 @@ public partial class App : Application
protected override async void OnStartup(StartupEventArgs e)
{
base.OnStartup(e);
+ ApplyAccessibilityTheme();
bool integration = e.Args.Contains("--integration-test"), self = e.Args.Contains("--self-test");
bool capture = e.Args.Contains("--capture-test");
bool control = e.Args.Contains("--control-test");
@@ -16,6 +17,7 @@ protected override async void OnStartup(StartupEventArgs e)
bool native = e.Args.Contains("--native-diagnostic");
bool notepad = e.Args.Contains("--notepad-test");
bool notepadGuide = e.Args.Contains("--notepad-guide-test");
+ DiagnosticLog.Record("desktop_started", new { version = "0.2.0" });
if (integration || self || capture || control || controlComponent || native || notepad || notepadGuide)
{
ShutdownMode = ShutdownMode.OnExplicitShutdown;
@@ -50,7 +52,67 @@ protected override async void OnStartup(StartupEventArgs e)
await NotepadTests.RunNative(notepadHandle, checks, value => stage = value,
notepadGuide ? InteractionMode.Guide : InteractionMode.Control);
}
- else { stage = "self-test"; SelfTests.Run(); checks.Add("desktop-safety"); NotepadTests.Run(checks); }
+ else
+ {
+ stage = "self-test";
+ SelfTests.Run();
+ checks.Add("desktop-safety");
+ CaptureTests.RunEvidenceChecks();
+ checks.Add("action-priority-evidence-bounds-browser-resource-identity");
+ NotepadTests.Run(checks);
+ stage = "camera-state-fixtures";
+ CameraRecoveryTests.Run();
+ checks.Add("camera-fixture-consent-state-readiness-boundaries");
+ CompactCameraTests.RunConsentChecks();
+ checks.Add("camera-request-consent-single-use-permission-scope-cancellation");
+ await CameraWindowDiscoveryTests.RunAsync();
+ checks.Add("teams-meeting-camera-discovery-not-home-bounded-ambiguous-cancel");
+ await CameraTargetRevalidationTests.RunAsync();
+ checks.Add("camera-revalidation-stable-control-exact-scope-freshness-already-enabled");
+ stage = "prompt-composer";
+ PromptTests.Run();
+ checks.Add("prompt-submit-keyboard-idle-ui");
+ CompanionModeTests.Run();
+ checks.Add("explicit-mode-radio-selection-idempotent-stop-revoke-unavailable");
+ CompactLayoutTests.Run();
+ checks.Add("compact-content-fit-permission-scope-stop-restart-warning-accessible");
+ CameraCompletionTests.Run();
+ checks.Add("camera-resolved-done-clears-task-no-action-fixture-distinction");
+ stage = "companion-position";
+ CompanionPositionTests.Run();
+ checks.Add("companion-pin-follow-drag-position-persistence-recovery-and-accessibility");
+ stage = "screen-task-loop";
+ await PromptTests.RunTaskLoopAsync();
+ checks.Add("screen-task-progress-checkpoint-outcomes-supersession");
+ stage = "screen-plan-segments";
+ await PlanTests.RunAsync(checks);
+ stage = "native-action-lifecycle";
+ await PromptTests.RunNativeLifecycleAsync();
+ checks.Add("bounded-native-action-unknown-late-return-no-retry");
+ checks.Add("visible-target-presentation-before-invocation-no-background-fallback");
+ stage = "guidance-client-deadline";
+ await PromptTests.RunClientDeadlineAsync();
+ checks.Add("guidance-client-remaining-freshness-cancellation");
+ stage = "speech-lifecycle";
+ await SpeechTests.RunAsync();
+ checks.Add("speech-drain-uncertainty-cancellation-input-feedback");
+ checks.Add("compact-voice-shared-draft-input-choice-stop-cancel-dismiss-no-auto-submit");
+ checks.Add("microphone-cancel-stop-timeout-late-ack-second-input-gating");
+ WhisperTests.Run();
+ checks.Add("whisper-bounded-memory-capture");
+ if (Environment.GetEnvironmentVariable("MSGUIDE_WHISPER_SYNTHETIC_TEST") == "1")
+ {
+ stage = "whisper-synthetic-input";
+ await WhisperTests.RunSyntheticAsync();
+ checks.Add("whisper-synthetic-transcription");
+ }
+ if (Environment.GetEnvironmentVariable("MSGUIDE_SPEECH_SYNTHETIC_TEST") == "1")
+ {
+ stage = "speech-synthetic-input";
+ await SpeechTests.RunSyntheticAsync();
+ checks.Add("speech-synthetic-memory-input");
+ }
+ }
}
catch (Exception ex)
{
@@ -73,7 +135,34 @@ await NotepadTests.RunNative(notepadHandle, checks, value => stage = value,
Shutdown(failure.Length == 0 ? 0 : 1);
return;
}
- MainWindow = new MainWindow();
- MainWindow.Show();
+ var mainWindow = new MainWindow();
+ MainWindow = mainWindow;
+ mainWindow.Show();
+ mainWindow.StartCompanionMode();
+ }
+
+ private void ApplyAccessibilityTheme()
+ {
+ if (!SystemParameters.HighContrast) return;
+ Resources["CanvasBrush"] = SystemColors.WindowBrush;
+ Resources["SurfaceBrush"] = SystemColors.ControlBrush;
+ Resources["SurfaceRaisedBrush"] = SystemColors.ControlBrush;
+ Resources["SurfaceHoverBrush"] = SystemColors.HighlightBrush;
+ Resources["InputBrush"] = SystemColors.WindowBrush;
+ Resources["BorderBrush"] = SystemColors.ActiveBorderBrush;
+ Resources["BorderStrongBrush"] = SystemColors.HighlightBrush;
+ Resources["TextBrush"] = SystemColors.WindowTextBrush;
+ Resources["MutedTextBrush"] = SystemColors.WindowTextBrush;
+ Resources["AccentBrush"] = SystemColors.HighlightBrush;
+ Resources["AccentStrongBrush"] = SystemColors.HighlightBrush;
+ Resources["AccentHoverBrush"] = SystemColors.HighlightBrush;
+ Resources["AccentSoftBrush"] = SystemColors.ControlBrush;
+ Resources["AccentTextBrush"] = SystemColors.HighlightTextBrush;
+ Resources["SuccessBrush"] = SystemColors.WindowTextBrush;
+ Resources["SuccessSoftBrush"] = SystemColors.WindowBrush;
+ Resources["WarningBrush"] = SystemColors.WindowTextBrush;
+ Resources["WarningSoftBrush"] = SystemColors.WindowBrush;
+ Resources["DangerBrush"] = SystemColors.WindowTextBrush;
+ Resources["DangerSoftBrush"] = SystemColors.WindowBrush;
}
}
\ No newline at end of file
diff --git a/desktop/AutomationEvidence.cs b/desktop/AutomationEvidence.cs
new file mode 100644
index 0000000..4566033
--- /dev/null
+++ b/desktop/AutomationEvidence.cs
@@ -0,0 +1,472 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Runtime.InteropServices;
+using System.Security.Cryptography;
+using System.Text;
+using System.Windows.Automation;
+
+namespace MSGuide.Desktop;
+
+internal static class AutomationEvidence
+{
+ internal const int ScanNodeLimit = 2000;
+ internal const int ScanDepthLimit = 64;
+ internal const int ScanMilliseconds = 3000;
+
+ private static readonly HashSet KnownAutomationIds =
+ [
+ "SystemSettings_CapabilityAccess_Camera_SystemGlobal_ToggleSwitch",
+ "SystemSettings_CapabilityAccess_Camera_UserGlobal_ToggleSwitch",
+ "MSTeams_8wekyb3d8bbwe_ToggleSwitch",
+ "SystemSettings_CapabilityAccess_Camera_ClassicGlobal_ToggleSwitch",
+ "more-options-header",
+ "AudioSettings",
+ "VideoSettings",
+ "open_camera_settings"
+ ];
+
+ internal static string Bounded(string? value, int maximum)
+ {
+ value = value?.Trim() ?? "";
+ return value[..Math.Min(value.Length, maximum)];
+ }
+
+ internal static string? Optional(string? value, int maximum)
+ {
+ var bounded = Bounded(value, maximum);
+ return bounded.Length == 0 ? null : bounded;
+ }
+
+ internal static bool IsKnownAutomationId(string automationId) =>
+ KnownAutomationIds.Contains(automationId);
+
+ internal static string VerifiedPage(IEnumerable automationIds)
+ {
+ var ids = automationIds.ToHashSet(StringComparer.Ordinal);
+ return ids.Contains("SystemSettings_CapabilityAccess_Camera_SystemGlobal_ToggleSwitch")
+ && ids.Contains("SystemSettings_CapabilityAccess_Camera_UserGlobal_ToggleSwitch")
+ ? "camera-privacy"
+ : ids.Contains("VideoSettings")
+ ? "teams-devices"
+ : "unknown";
+ }
+
+ internal static string TargetId(WindowChoice window, string role, string label, double[] box,
+ string automationId, string frameworkId, int providerProcessId, IReadOnlyList? runtimeId)
+ {
+ var canonical = new StringBuilder(512)
+ .Append("uia-v1|").Append(window.Id).Append('|').Append(window.ClassName)
+ .Append('|').Append(role).Append('|').Append(label)
+ .Append('|').Append(automationId).Append('|').Append(frameworkId)
+ .Append('|').Append(providerProcessId).Append('|');
+ foreach (double coordinate in box)
+ canonical.Append(coordinate.ToString("R", CultureInfo.InvariantCulture)).Append(',');
+ canonical.Append('|');
+ if (runtimeId is not null)
+ foreach (int part in runtimeId) canonical.Append(part).Append(',');
+ var digest = SHA256.HashData(Encoding.UTF8.GetBytes(canonical.ToString()));
+ return "uia-" + Convert.ToHexString(digest.AsSpan(0, 12)).ToLowerInvariant();
+ }
+
+ internal static string? ControlId(WindowChoice window, int providerProcessId, IReadOnlyList? runtimeId)
+ {
+ if (providerProcessId <= 0 || runtimeId is not { Count: > 0 and <= 64 }) return null;
+ string identity = string.Join("|", "control-v1", window.Id, window.ClassName,
+ providerProcessId.ToString(CultureInfo.InvariantCulture), string.Join(",", runtimeId));
+ return "control-" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(identity))).ToLowerInvariant();
+ }
+
+ internal static string? ResourceId(WindowChoice window, string title, ElementInfo[] elements)
+ {
+ // Generic UIA document trees do not prove a file/site identity. Use an explicit resource handoff.
+ if (string.IsNullOrWhiteSpace(title) || elements.Any(e => e.Role == "document")) return null;
+ return "resource-" + ValueDigest(string.Join("|", window.Id, window.ClassName, title));
+ }
+
+ internal static CacheRequest CaptureCache(bool details = false)
+ {
+ var cache = new CacheRequest { TreeScope = TreeScope.Element };
+ AutomationProperty[] properties =
+ [
+ AutomationElement.IsPasswordProperty, AutomationElement.IsOffscreenProperty,
+ AutomationElement.IsEnabledProperty, AutomationElement.ProcessIdProperty,
+ AutomationElement.ControlTypeProperty, AutomationElement.BoundingRectangleProperty
+ ];
+ foreach (var property in properties) cache.Add(property);
+ if (details)
+ {
+ AutomationProperty[] metadata =
+ [
+ AutomationElement.NameProperty, AutomationElement.AutomationIdProperty,
+ AutomationElement.FrameworkIdProperty, AutomationElement.RuntimeIdProperty,
+ AutomationElement.HelpTextProperty, AutomationElement.ItemStatusProperty,
+ AutomationElement.IsValuePatternAvailableProperty, AutomationElement.IsTogglePatternAvailableProperty,
+ AutomationElement.IsInvokePatternAvailableProperty, AutomationElement.IsSelectionItemPatternAvailableProperty,
+ AutomationElement.IsExpandCollapsePatternAvailableProperty, AutomationElement.IsScrollPatternAvailableProperty
+ ];
+ foreach (var property in metadata) cache.Add(property);
+ }
+ return cache;
+ }
+
+ internal static bool IsSupportedBrowser(WindowChoice window)
+ {
+ if (!window.ClassName.StartsWith("Chrome_WidgetWin_", StringComparison.Ordinal)) return false;
+ try
+ {
+ using var process = Process.GetProcessById((int)window.ProcessId);
+ return process.ProcessName.Equals("msedge", StringComparison.OrdinalIgnoreCase)
+ || process.ProcessName.Equals("chrome", StringComparison.OrdinalIgnoreCase);
+ }
+ catch (Exception ex) when (ex is ArgumentException or InvalidOperationException
+ or System.ComponentModel.Win32Exception)
+ {
+ DiagnosticLog.Record("browser_identity_unavailable", new { errorType = ex.GetType().Name });
+ return false;
+ }
+ }
+
+ internal static string? BrowserResourceId(WindowChoice window, string address)
+ {
+ if (address.Length > 2048 || !Uri.TryCreate(address, UriKind.Absolute, out var uri)
+ || uri.Scheme is not ("https" or "http") || uri.UserInfo.Length != 0
+ || string.IsNullOrWhiteSpace(uri.Host))
+ return null;
+ return "browser-" + ValueDigest(string.Join("|", window.Id, window.ClassName, uri.AbsoluteUri));
+ }
+
+ internal static bool IsBrowserAddressControl(string role, string name, bool insideDocument) =>
+ !insideDocument && role == "edit" && name is "Address and search bar" or "Address bar";
+
+ internal static string? ReadBrowserResourceId(WindowChoice window, CancellationToken token)
+ {
+ try { return ReadBrowserResourceCore(window, token); }
+ catch (OperationCanceledException) { throw; }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException
+ or COMException or UnauthorizedAccessException or ArgumentException)
+ {
+ DiagnosticLog.Record("browser_resource_inspection_failed", new { errorType = ex.GetType().Name });
+ return null;
+ }
+ }
+
+ private static string? ReadBrowserResourceCore(WindowChoice window, CancellationToken token)
+ {
+ if (!IsSupportedBrowser(window) || !window.Matches()
+ || !Native.GetWindowRect(window.Handle, out var rect)) return null;
+ var privacy = CaptureCache();
+ var details = CaptureCache(details: true);
+ var root = AutomationElement.FromHandle(window.Handle).GetUpdatedCache(privacy);
+ if (root.Cached.ProcessId != (int)window.ProcessId) return null;
+ var walker = TreeWalker.RawViewWalker;
+ var clock = Stopwatch.StartNew();
+ int visited = 0, documents = 0, addresses = 0;
+ bool incomplete = false;
+ string? resource = null;
+ AutomationElement? addressControl = null;
+ void Walk(AutomationElement node, int depth)
+ {
+ token.ThrowIfCancellationRequested();
+ if (++visited > ScanNodeLimit || depth > ScanDepthLimit || clock.ElapsedMilliseconds >= ScanMilliseconds)
+ { incomplete = true; return; }
+ var value = node.Cached;
+ if (value.IsPassword || value.IsOffscreen) return;
+ if (value.ControlType == ControlType.Document)
+ {
+ if (Safety.AutomationBox(value.BoundingRectangle, rect) is not null) documents++;
+ return; // Never accept an address-like field supplied by web content.
+ }
+ if (value.ControlType == ControlType.TabItem) return;
+ if (value.ControlType == ControlType.Edit && value.IsEnabled
+ && Safety.AutomationBox(value.BoundingRectangle, rect) is not null)
+ {
+ var field = node.GetUpdatedCache(details);
+ if (!field.Cached.IsPassword && !field.Cached.IsOffscreen
+ && IsBrowserAddressControl("edit", field.Cached.Name, insideDocument: false))
+ {
+ addresses++;
+ addressControl = field;
+ if (!field.Current.IsPassword
+ && field.TryGetCurrentPattern(ValuePattern.Pattern, out var pattern)
+ && pattern is ValuePattern address)
+ resource = BrowserResourceId(window, address.Current.Value);
+ }
+ }
+ var child = walker.GetFirstChild(node, privacy);
+ while (child is not null && !incomplete)
+ {
+ Walk(child, depth + 1);
+ if (incomplete) break;
+ child = walker.GetNextSibling(child, privacy);
+ }
+ }
+ Walk(root, 0);
+ token.ThrowIfCancellationRequested();
+ if (incomplete || clock.ElapsedMilliseconds >= ScanMilliseconds
+ || addresses != 1 || documents != 1 || addressControl is null || resource is null
+ || !window.Matches() || !Native.GetWindowRect(window.Handle, out var after) || !rect.Same(after))
+ return null;
+ var current = addressControl.Current;
+ return !current.IsPassword && !current.IsOffscreen && current.IsEnabled
+ && addressControl.TryGetCurrentPattern(ValuePattern.Pattern, out var lastPattern)
+ && lastPattern is ValuePattern last
+ && BrowserResourceId(window, last.Current.Value) == resource ? resource : null;
+ }
+
+ internal static bool ResourceMatches(WindowChoice window, string expected, CancellationToken token) =>
+ expected.StartsWith("browser-", StringComparison.Ordinal)
+ ? ReadBrowserResourceId(window, token) == expected
+ : ResourceId(window, Native.Title(window.Handle), []) == expected;
+
+ internal static string? ToggleState(AutomationElement element)
+ {
+ if (!element.TryGetCurrentPattern(TogglePattern.Pattern, out var pattern)
+ || pattern is not TogglePattern toggle) return null;
+ return toggle.Current.ToggleState switch
+ {
+ System.Windows.Automation.ToggleState.On => "on",
+ System.Windows.Automation.ToggleState.Off => "off",
+ System.Windows.Automation.ToggleState.Indeterminate => "indeterminate",
+ _ => null
+ };
+ }
+
+ internal static string ValueDigest(string value) =>
+ Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))).ToLowerInvariant();
+
+ internal sealed record ActionMetadata(string? Name, bool? IsReadOnly = null,
+ string? ValueHash = null, int? ValueLength = null, bool? IsSelected = null,
+ string[]? ScrollDirections = null, double? HorizontalScrollPercent = null,
+ double? VerticalScrollPercent = null);
+
+ internal static ActionMetadata ReadAction(AutomationElement element, bool cachedPatterns = false)
+ {
+ var current = cachedPatterns ? element.Cached : element.Current;
+ if (current.IsPassword || current.IsOffscreen || !current.IsEnabled) return new(null);
+ bool TryPattern(AutomationPattern pattern, AutomationProperty available, out object value)
+ {
+ value = null!;
+ return (!cachedPatterns || element.GetCachedPropertyValue(available) is true)
+ && element.TryGetCurrentPattern(pattern, out value);
+ }
+ if (TryPattern(ValuePattern.Pattern, AutomationElement.IsValuePatternAvailableProperty, out var valuePattern)
+ && valuePattern is ValuePattern value && !value.Current.IsReadOnly)
+ {
+ var live = element.Current;
+ if (live.IsPassword || live.IsOffscreen || !live.IsEnabled) return new(null);
+ string text = value.Current.Value;
+ // Full-field replacement only. Large/document editors need a dedicated adapter.
+ return text.Length <= 1000
+ ? new("set_value", false, ValueDigest(text), text.Length)
+ : new(null);
+ }
+ bool toggle = TryPattern(TogglePattern.Pattern, AutomationElement.IsTogglePatternAvailableProperty, out _);
+ bool invoke = TryPattern(InvokePattern.Pattern, AutomationElement.IsInvokePatternAvailableProperty, out _);
+ bool select = TryPattern(SelectionItemPattern.Pattern, AutomationElement.IsSelectionItemPatternAvailableProperty, out var selection);
+ bool expandable = TryPattern(ExpandCollapsePattern.Pattern, AutomationElement.IsExpandCollapsePatternAvailableProperty, out var pattern);
+ string? state = pattern is ExpandCollapsePattern expand
+ ? expand.Current.ExpandCollapseState.ToString().ToLowerInvariant()
+ : null;
+ string? action = ActionName(toggle, invoke, select, expandable, state);
+ if (action is not null)
+ return new(action, IsSelected: selection is SelectionItemPattern item ? item.Current.IsSelected : null);
+ if (TryPattern(ScrollPattern.Pattern, AutomationElement.IsScrollPatternAvailableProperty, out var scrollPattern)
+ && scrollPattern is ScrollPattern scroll)
+ {
+ double horizontal = scroll.Current.HorizontalScrollPercent;
+ double vertical = scroll.Current.VerticalScrollPercent;
+ var directions = ScrollDirections(horizontal, vertical);
+ if (directions.Length > 0)
+ return new("scroll", ScrollDirections: directions,
+ HorizontalScrollPercent: horizontal, VerticalScrollPercent: vertical);
+ }
+ return new(null);
+ }
+
+ internal static string? Action(AutomationElement element) => ReadAction(element).Name;
+
+ internal static string[] ScrollDirections(double horizontal, double vertical)
+ {
+ var directions = new List(4);
+ if (horizontal is > 0 and <= 100) directions.Add("left");
+ if (horizontal is >= 0 and < 100) directions.Add("right");
+ if (vertical is > 0 and <= 100) directions.Add("up");
+ if (vertical is >= 0 and < 100) directions.Add("down");
+ return directions.ToArray();
+ }
+
+ internal static string? ActionName(
+ bool toggle, bool invoke, bool select, bool expandable, string? expandState) =>
+ toggle ? "toggle"
+ : invoke ? "invoke"
+ : select ? "select"
+ : expandable && expandState == "collapsed" ? "expand"
+ : expandable && expandState == "expanded" ? "collapse"
+ : null;
+
+ internal static bool MatchesTargetId(
+ WindowChoice window, Native.RECT rect, AutomationElement element, string expected)
+ {
+ var value = element.Current;
+ if (value.IsPassword || value.IsOffscreen) return false;
+ var box = Safety.AutomationBox(value.BoundingRectangle, rect);
+ if (box is null) return false;
+ string name = Bounded(value.Name, 256);
+ string automationId = Bounded(value.AutomationId, 128);
+ if (name.Length == 0 && !IsKnownAutomationId(automationId)) return false;
+ int[]? runtimeId = null;
+ try { runtimeId = element.GetRuntimeId(); }
+ catch (Exception ex) when (
+ ex is ElementNotAvailableException or InvalidOperationException or COMException) { }
+ if (ControlId(window, value.ProcessId, runtimeId) is null) return false;
+ string role = value.ControlType.ProgrammaticName
+ .Replace("ControlType.", "").ToLowerInvariant();
+ string label = name.Length > 0 ? name : automationId;
+ return string.Equals(TargetId(
+ window, role, label, box, automationId, Bounded(value.FrameworkId, 64),
+ value.ProcessId, runtimeId), expected, StringComparison.Ordinal);
+ }
+
+ internal static AutomationElement? FindUniqueTarget(
+ WindowChoice window, Native.RECT rect, string targetId, string label,
+ string? automationId, CancellationToken cancellationToken, string? resourceId = null)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var privacy = CaptureCache();
+ var details = CaptureCache(details: true);
+ var root = AutomationElement.FromHandle(window.Handle).GetUpdatedCache(privacy);
+ if (root.Cached.ProcessId != (int)window.ProcessId) return null;
+ if (resourceId is not null && !ResourceMatches(window, resourceId, cancellationToken)) return null;
+ var walker = TreeWalker.RawViewWalker;
+ var clock = Stopwatch.StartNew();
+ int visited = 0;
+ bool incomplete = false, duplicate = false;
+ AutomationElement? match = null;
+ void Walk(AutomationElement node, int depth)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (++visited > ScanNodeLimit || depth > ScanDepthLimit || clock.ElapsedMilliseconds >= ScanMilliseconds)
+ {
+ incomplete = true;
+ return;
+ }
+ var value = node.Cached;
+ if (resourceId is not null && !resourceId.StartsWith("browser-", StringComparison.Ordinal)
+ && value.ControlType == ControlType.Document)
+ {
+ incomplete = true;
+ return;
+ }
+ if (value.IsPassword || value.IsOffscreen) return;
+ var named = node.GetUpdatedCache(details).Cached;
+ if (named.IsPassword || named.IsOffscreen) return;
+ bool candidate = string.IsNullOrWhiteSpace(automationId)
+ ? named.Name == label : named.AutomationId == automationId;
+ if (candidate && MatchesTargetId(window, rect, node, targetId))
+ {
+ if (match is not null) { duplicate = true; return; }
+ match = node;
+ }
+ var child = walker.GetFirstChild(node, privacy);
+ while (child is not null && !incomplete && !duplicate)
+ {
+ Walk(child, depth + 1);
+ if (incomplete || duplicate) break;
+ child = walker.GetNextSibling(child, privacy);
+ }
+ }
+ // Unlike FindAll, traversal has node/depth/time ceilings. Individual COM calls
+ // can still hang; the action caller contains one late native worker.
+ Walk(root, 0);
+ return incomplete || duplicate || clock.ElapsedMilliseconds >= ScanMilliseconds
+ || resourceId is not null && !ResourceMatches(window, resourceId, cancellationToken) ? null : match;
+ }
+
+ internal static AutomationProbeDiagnostic Probe(WindowChoice window, Native.RECT rect, CancellationToken ct)
+ {
+ int visited = 0, inBounds = 0, enabled = 0, disabled = 0, crossProcess = 0, toggles = 0;
+ bool truncated = false;
+ var roles = new Dictionary(StringComparer.Ordinal);
+ var knownControls = new Dictionary(StringComparer.Ordinal);
+ var clock = Stopwatch.StartNew();
+ try
+ {
+ var root = AutomationElement.FromHandle(window.Handle);
+ if (root.Current.ProcessId != (int)window.ProcessId)
+ return Diagnostic(false, 0, 0, 0, 0, 0, 0, false,
+ "root-identity-changed", roles, knownControls);
+ var walker = TreeWalker.RawViewWalker;
+ void Walk(AutomationElement node, int depth)
+ {
+ ct.ThrowIfCancellationRequested();
+ if (visited >= 800 || depth > 18 || clock.ElapsedMilliseconds >= 3000)
+ {
+ truncated = true;
+ return;
+ }
+ visited++;
+ var value = node.Current;
+ if (value.IsPassword) return;
+ string automationId = Bounded(value.AutomationId, 128);
+ if (IsKnownAutomationId(automationId))
+ {
+ string? toggleState = null;
+ try { toggleState = ToggleState(node); }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or COMException) { }
+ knownControls[automationId] = new(automationId, value.IsEnabled,
+ value.IsOffscreen, toggleState);
+ }
+ if (value.IsOffscreen) return;
+ if (Safety.AutomationBox(value.BoundingRectangle, rect) is not null)
+ {
+ inBounds++;
+ if (value.IsEnabled) enabled++; else disabled++;
+ if (value.ProcessId != (int)window.ProcessId) crossProcess++;
+ string role = value.ControlType.ProgrammaticName.Replace("ControlType.", "").ToLowerInvariant();
+ roles[role] = roles.GetValueOrDefault(role) + 1;
+ try
+ {
+ if (node.TryGetCurrentPattern(TogglePattern.Pattern, out _)) toggles++;
+ }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or COMException) { }
+ }
+ var child = walker.GetFirstChild(node);
+ while (child is not null)
+ {
+ Walk(child, depth + 1);
+ if (truncated) break;
+ child = walker.GetNextSibling(child);
+ }
+ }
+ Walk(root, 0);
+ return Diagnostic(true, visited, inBounds, enabled, disabled, crossProcess, toggles,
+ truncated, "completed", roles, knownControls);
+ }
+ catch (OperationCanceledException) { throw; }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException
+ or COMException or UnauthorizedAccessException)
+ {
+ return Diagnostic(true, visited, inBounds, enabled, disabled, crossProcess, toggles,
+ truncated, "provider-error", roles, knownControls);
+ }
+ }
+
+ private static AutomationProbeDiagnostic Diagnostic(bool rootMatched, int visited, int inBounds,
+ int enabled, int disabled, int crossProcess, int toggles, bool truncated, string outcome,
+ IReadOnlyDictionary roles,
+ IReadOnlyDictionary knownControls)
+ {
+ var controls = knownControls.Values.OrderBy(control => control.AutomationId).ToArray();
+ string page = VerifiedPage(knownControls.Keys);
+ return new(rootMatched, visited, inBounds, enabled, disabled, crossProcess, toggles,
+ truncated, outcome, page, roles, controls);
+ }
+}
+
+public sealed record AutomationProbeDiagnostic(bool RootMatched, int NodesVisited, int InBoundsElements,
+ int EnabledElements, int DisabledElements, int CrossProcessElements, int TogglePatterns,
+ bool Truncated, string Outcome, string VerifiedPage, IReadOnlyDictionary Roles,
+ KnownControlDiagnostic[] KnownControls);
+
+public sealed record KnownControlDiagnostic(string AutomationId, bool IsEnabled, bool IsOffscreen,
+ string? ToggleState);
diff --git a/desktop/CameraCompletionTests.cs b/desktop/CameraCompletionTests.cs
new file mode 100644
index 0000000..e77e5f7
--- /dev/null
+++ b/desktop/CameraCompletionTests.cs
@@ -0,0 +1,83 @@
+using System.Windows;
+using System.Windows.Automation;
+using System.Windows.Controls;
+
+namespace MSGuide.Desktop;
+
+internal static class CameraCompletionTests
+{
+ internal static void Run()
+ {
+ foreach (var finding in new[]
+ {
+ CameraVerificationFinding.Ready, CameraVerificationFinding.AlreadyReady,
+ CameraVerificationFinding.NeedsReinitialization, CameraVerificationFinding.Unresolved
+ })
+ foreach (bool fixture in new[] { false, true })
+ {
+ var main = new MainWindow(new SpeechService(), new CompanionPosition());
+ try { main.CheckCameraCompletion(finding, fixture); }
+ finally { main.Close(); }
+ }
+ }
+}
+
+public partial class MainWindow
+{
+ internal void CheckCameraCompletion(CameraVerificationFinding finding, bool fixture)
+ {
+ loaded = true;
+ try
+ {
+ var sensing = new CompactCameraTests.Sensing();
+ cameraRecoverySensing = sensing;
+ PromptBox.Text = "Check my Teams camera";
+ cameraRecovery = new CameraRecoverySession(CameraRecoveryInteractionMode.Guide);
+ cameraRecovery.Start();
+ cameraRecovery.ChooseTeamsWindow("synthetic-completion", "Synthetic meeting");
+ cameraRecovery.ApplyTeamsObservation(new("synthetic-completion", TeamsCameraFinding.CameraOn, ""));
+ UpdateCameraRecoveryUi();
+ IntegrationTests.Require(!cameraRecovery.CanComplete && !CameraDoneButton.IsEnabled
+ && CameraDoneButton.Visibility == Visibility.Collapsed);
+ CameraDone_Click(this, new RoutedEventArgs());
+ IntegrationTests.Require(cameraRecovery.State == CameraRecoveryState.NeedsLocalVerification);
+
+ bool verified = finding is CameraVerificationFinding.Ready or CameraVerificationFinding.AlreadyReady;
+ cameraRecovery.ApplyVerification(new("synthetic-completion", finding, verified, "Synthetic evidence.",
+ IsFixture: fixture, Reinitialized: finding == CameraVerificationFinding.Ready));
+ UpdateCameraRecoveryUi();
+ IntegrationTests.Require(cameraRecovery.CanComplete == verified
+ && CameraDoneButton.IsEnabled == verified
+ && (CameraDoneButton.Visibility == Visibility.Visible) == verified);
+ if (!verified)
+ {
+ IntegrationTests.Require(!CameraStateText.Text.StartsWith("Resolved", StringComparison.Ordinal));
+ return;
+ }
+ IntegrationTests.Require(ReferenceEquals(CameraDoneButton.Style, FindResource("PrimaryButtonStyle"))
+ && ReferenceEquals(CameraStartButton.Style, FindResource("QuietButtonStyle")));
+ if (fixture)
+ IntegrationTests.Require(CameraStateText.Text.StartsWith("Fixture complete", StringComparison.Ordinal)
+ && AutomationProperties.GetName(CameraDoneButton).Contains("simulated")
+ && !CameraStateText.Text.Contains("Resolved"));
+ else
+ {
+ IntegrationTests.Require(CameraStateText.Text == "Resolved · camera ready"
+ && AutomationProperties.GetName(CameraDoneButton).Contains("resolved"));
+ if (finding == CameraVerificationFinding.AlreadyReady)
+ IntegrationTests.Require(CameraStepText.Text.Contains("No change was needed."));
+ }
+ CameraDoneButton.RaiseEvent(new RoutedEventArgs(Button.ClickEvent));
+ IntegrationTests.Require(cameraRecovery.CanStart && !cameraRecovery.CanComplete
+ && !cameraRecovery.CameraOnRequestAuthorized && cameraRecovery.Target is null
+ && PromptBox.Text.Length == 0 && companion.Prompt.DraftControl.Text.Length == 0
+ && CameraRecoveryCard.Visibility == Visibility.Collapsed
+ && CameraDoneButton.Visibility == Visibility.Collapsed
+ && sensing.Actions.Count == 0 && sensing.Restarts == 0
+ && !speech.Busy && !companion.Prompt.SettingsVisible);
+ CameraDone_Click(this, new RoutedEventArgs());
+ IntegrationTests.Require(cameraRecovery.CanStart && sensing.Actions.Count == 0);
+ }
+ finally { loaded = false; }
+ }
+}
diff --git a/desktop/CameraPermissionEvidence.cs b/desktop/CameraPermissionEvidence.cs
new file mode 100644
index 0000000..467628b
--- /dev/null
+++ b/desktop/CameraPermissionEvidence.cs
@@ -0,0 +1,42 @@
+namespace MSGuide.Desktop;
+
+internal enum CameraPermissionState { Unknown, Off, On, Managed }
+
+internal sealed record CameraPermissionEvidence(
+ CameraPermissionState State, CameraRecoveryTargetKind Scope, string Detail)
+{
+ public static CameraPermissionEvidence FromConsent(
+ bool policyDenied, string? device, string? apps, string? teams)
+ {
+ if (policyDenied)
+ return new(CameraPermissionState.Managed, CameraRecoveryTargetKind.Unknown,
+ "Camera access is blocked by policy. MSGuide will not override it.");
+
+ foreach (var (value, scope) in new[]
+ {
+ (device, CameraRecoveryTargetKind.DeviceCameraPermission),
+ (apps, CameraRecoveryTargetKind.AppCameraPermission),
+ (teams, CameraRecoveryTargetKind.PackagedTeamsPermission)
+ })
+ {
+ if (value == "Deny")
+ return new(CameraPermissionState.Off, scope, DescribeBlock(scope));
+ if (value != "Allow")
+ return new(CameraPermissionState.Unknown, scope,
+ "Windows camera permissions could not all be established. Inspect Camera settings before attempting a camera action.");
+ }
+ return new(CameraPermissionState.On, CameraRecoveryTargetKind.Unknown,
+ "Device, app, and Microsoft Teams camera permissions are on.");
+ }
+
+ public static string DescribeBlock(CameraRecoveryTargetKind scope) => scope switch
+ {
+ CameraRecoveryTargetKind.DeviceCameraPermission =>
+ "Windows device-wide Camera access is off. This blocks Teams even when its own permission is on. Enabling it permits camera access for other apps that already have permission; separate approval is required.",
+ CameraRecoveryTargetKind.AppCameraPermission =>
+ "Let apps access your camera is off. Enabling it affects other permitted apps for this user, not only Teams; separate approval is required.",
+ CameraRecoveryTargetKind.PackagedTeamsPermission =>
+ "The individual Microsoft Teams camera permission is off.",
+ _ => throw new ArgumentOutOfRangeException(nameof(scope))
+ };
+}
diff --git a/desktop/CameraRecoverySession.cs b/desktop/CameraRecoverySession.cs
new file mode 100644
index 0000000..09e2ac6
--- /dev/null
+++ b/desktop/CameraRecoverySession.cs
@@ -0,0 +1,842 @@
+namespace MSGuide.Desktop;
+
+internal enum CameraRecoveryState
+{
+ Idle,
+ NeedsTeamsObservation,
+ Diagnosis,
+ NeedsCameraSettings,
+ NeedsSettingsObservation,
+ VerifiedTarget,
+ PermissionObservedOn,
+ NeedsLocalVerification,
+ NeedsCameraReinitialization,
+ Ready,
+ FixtureComplete,
+ ReadOnlyAssessment,
+ ControlRevalidationRequired,
+ WrongSettingsPage,
+ ManagedOrDisabled,
+ AlreadyOnOrWrongCause,
+ StaleOrMoved,
+ UnresolvedAfterPermission,
+ Unsupported,
+ Cancelled
+}
+
+internal enum TeamsCameraFinding
+{
+ CameraOff,
+ CameraOn,
+ PermissionMayBeOff,
+ PermissionAlreadyOnOrDifferentCause,
+ ManagedOrDisabled,
+ StaleOrMoved,
+ Unsupported
+}
+
+internal enum CameraSettingsFinding
+{
+ PermissionOff,
+ PermissionOn,
+ WrongPage,
+ ManagedOrDisabled,
+ StaleOrMoved,
+ Unsupported
+}
+
+internal enum CameraVerificationFinding
+{
+ Ready,
+ AlreadyReady,
+ NeedsReinitialization,
+ Unresolved,
+ StaleOrMoved,
+ Unsupported
+}
+
+internal enum CameraRecoverySensingMode
+{
+ UnsupportedFallback,
+ Fixture,
+ Connected
+}
+
+internal enum CameraRecoveryInteractionMode
+{
+ Guide,
+ Control
+}
+
+internal enum CameraRecoveryTargetKind
+{
+ Unknown,
+ TeamsCameraButton,
+ PackagedTeamsPermission,
+ DeviceCameraPermission,
+ AppCameraPermission
+}
+
+internal enum TeamsRestartFinding
+{
+ Restarted,
+ StaleOrMoved,
+ Unsupported,
+ Failed
+}
+
+internal enum CameraSettingsObservationSource
+{
+ Unknown,
+ ControlsOnly,
+ PrivateVisual,
+ Fixture
+}
+
+internal enum CameraSettingsPage
+{
+ Unknown,
+ CameraPrivacy,
+ Other
+}
+
+internal static class CameraRecoveryPinnedTargets
+{
+ private const string TeamsCameraShortcut = "Ctrl+Shift+O";
+ public const string TeamsVideoSettings = "VideoSettings";
+ public const string PackagedTeamsCameraToggle = "MSTeams_8wekyb3d8bbwe_ToggleSwitch";
+ public const string DeviceCameraToggle = "SystemSettings_CapabilityAccess_Camera_SystemGlobal_ToggleSwitch";
+ public const string AppCameraToggle = "SystemSettings_CapabilityAccess_Camera_UserGlobal_ToggleSwitch";
+ public const string TeamsTurnCameraOn = "Turn camera on";
+ public const string TeamsTurnCameraOff = "Turn camera off";
+ public const string TeamsCameraToggle = "Camera";
+ public const string FixturePreparation =
+ "Fixture mode simulates a Teams camera button that starts off.";
+
+ public static bool IsTeamsTurnCameraOn(string? label) =>
+ MatchesTeamsCommand(label, TeamsTurnCameraOn);
+
+ public static bool IsTeamsTurnCameraOff(string? label) =>
+ MatchesTeamsCommand(label, TeamsTurnCameraOff);
+
+ public static bool IsTeamsCameraToggle(string? label) =>
+ string.Equals(label, TeamsCameraToggle, StringComparison.Ordinal);
+
+ internal static bool IsTeamsCameraElement(ElementInfo element) =>
+ element.Role is "button" or "checkbox" && !element.IsPassword && !element.IsOffscreen
+ && (IsTeamsTurnCameraOn(element.Label) || IsTeamsTurnCameraOff(element.Label)
+ || IsTeamsCameraToggle(element.Label) && element.ToggleState is "off" or "on");
+
+ public static bool IsPermission(CameraRecoveryTargetKind kind) =>
+ kind is CameraRecoveryTargetKind.PackagedTeamsPermission
+ or CameraRecoveryTargetKind.DeviceCameraPermission
+ or CameraRecoveryTargetKind.AppCameraPermission;
+
+ public static bool IsPermissionTarget(string? automationId, CameraRecoveryTargetKind kind) =>
+ kind switch
+ {
+ CameraRecoveryTargetKind.PackagedTeamsPermission => automationId == PackagedTeamsCameraToggle,
+ CameraRecoveryTargetKind.DeviceCameraPermission => automationId == DeviceCameraToggle,
+ CameraRecoveryTargetKind.AppCameraPermission => automationId == AppCameraToggle,
+ _ => false
+ };
+
+ private static bool MatchesTeamsCommand(string? label, string command)
+ {
+ if (string.Equals(label, command, StringComparison.Ordinal)) return true;
+ return string.Equals(
+ label, $"{command} ({TeamsCameraShortcut})", StringComparison.Ordinal);
+ }
+}
+
+internal sealed record TeamsCameraObservation(
+ string WindowId, TeamsCameraFinding Finding, string Detail, CameraRecoveryTarget? Target = null,
+ CameraRecoveryTargetKind PermissionScope = CameraRecoveryTargetKind.Unknown);
+internal sealed record CameraRecoveryTarget(
+ string ObservationId, string Label, string? AutomationId = null,
+ CameraRecoveryTargetKind Kind = CameraRecoveryTargetKind.Unknown);
+internal sealed record CameraSettingsObservation(
+ CameraSettingsFinding Finding, string Detail, CameraRecoveryTarget? Target = null,
+ CameraSettingsObservationSource Source = CameraSettingsObservationSource.Unknown,
+ bool ProbeValidated = false, CameraSettingsPage Page = CameraSettingsPage.Unknown);
+internal sealed record CameraVerificationResult(
+ string WindowId, CameraVerificationFinding Finding, bool LocalVerifierPassed, string Detail,
+ bool IsFixture = false, bool Reinitialized = false);
+internal sealed record CameraTargetPresentation(bool Shown, string Detail);
+internal sealed record CameraTargetControlResult(
+ bool Invoked, bool OutcomeKnown, string Detail, bool StateAlreadySatisfied = false);
+internal sealed record TeamsRestartResult(
+ TeamsRestartFinding Finding, string Detail, WindowChoice? ReopenedWindow = null);
+
+internal interface ICameraRecoverySensing
+{
+ CameraRecoverySensingMode Mode { get; }
+ void Reset();
+ // Stay rooted to window.Handle, but do not require WebView descendants to share its process ID.
+ Task ObserveTeamsAsync(WindowChoice window, CancellationToken cancellationToken);
+ // Supported findings must identify a disclosed observation source that has passed a live probe.
+ Task ObserveSettingsAsync(CancellationToken cancellationToken);
+ Task VerifyTeamsAsync(WindowChoice window, CancellationToken cancellationToken);
+ Task ShowTargetAsync(CameraRecoveryTarget target, CancellationToken cancellationToken);
+}
+
+internal interface ICameraRecoveryControl
+{
+ Task ActivateTargetAsync(
+ CameraRecoveryTarget target, CancellationToken cancellationToken);
+ Task RestartTeamsAsync(
+ WindowChoice window, CancellationToken cancellationToken);
+}
+
+internal sealed class PendingCameraRecoverySensing : ICameraRecoverySensing
+{
+ private const string Pending =
+ "Controls-only camera sensing is not available in this branch yet. This action did not take a screenshot or send data.";
+
+ public CameraRecoverySensingMode Mode => CameraRecoverySensingMode.UnsupportedFallback;
+ public void Reset() { }
+
+ public Task ObserveTeamsAsync(WindowChoice window, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return Task.FromResult(new TeamsCameraObservation(window.Id, TeamsCameraFinding.Unsupported, Pending));
+ }
+
+ public Task ObserveSettingsAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return Task.FromResult(new CameraSettingsObservation(CameraSettingsFinding.Unsupported, Pending));
+ }
+
+ public Task VerifyTeamsAsync(WindowChoice window, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return Task.FromResult(new CameraVerificationResult(
+ window.Id, CameraVerificationFinding.Unsupported, false, Pending));
+ }
+
+ public Task ShowTargetAsync(
+ CameraRecoveryTarget target, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return Task.FromResult(new CameraTargetPresentation(false,
+ "A verified target presenter has not been connected. No outline or click was attempted."));
+ }
+}
+
+internal sealed class FixtureCameraRecoverySensing : ICameraRecoverySensing, ICameraRecoveryControl, ICameraWindowDiscovery
+{
+ private int teamsObservations;
+ private int settingsObservations;
+ private int verificationAttempts;
+ private bool cameraEnabled;
+ private bool permissionFlow;
+
+ public CameraRecoverySensingMode Mode => CameraRecoverySensingMode.Fixture;
+
+ public Task InspectCameraSurfaceAsync(WindowChoice window, CancellationToken ct)
+ {
+ ct.ThrowIfCancellationRequested();
+ return Task.FromResult(CameraSurfaceFinding.MeetingCamera);
+ }
+
+ public void Reset()
+ {
+ teamsObservations = 0;
+ settingsObservations = 0;
+ verificationAttempts = 0;
+ cameraEnabled = false;
+ permissionFlow = false;
+ }
+
+ public Task ObserveTeamsAsync(
+ WindowChoice window, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ teamsObservations++;
+ if (!cameraEnabled && teamsObservations == 1)
+ return Task.FromResult(new TeamsCameraObservation(
+ window.Id, TeamsCameraFinding.CameraOff, "Fixture: Teams camera is off.",
+ new CameraRecoveryTarget(
+ "fixture-teams-camera-1", CameraRecoveryPinnedTargets.TeamsTurnCameraOn,
+ "fixture-teams-camera-toggle", CameraRecoveryTargetKind.TeamsCameraButton)));
+ cameraEnabled = true;
+ return Task.FromResult(new TeamsCameraObservation(
+ window.Id, TeamsCameraFinding.CameraOn, "Fixture: Teams camera is on."));
+ }
+
+ public Task ObserveSettingsAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ permissionFlow = true;
+ settingsObservations++;
+ return Task.FromResult(settingsObservations == 1
+ ? new CameraSettingsObservation(CameraSettingsFinding.PermissionOff,
+ "Fixture: packaged Teams permission off.",
+ new CameraRecoveryTarget("fixture-settings-1", "Microsoft Teams Currently in use",
+ CameraRecoveryPinnedTargets.PackagedTeamsCameraToggle,
+ CameraRecoveryTargetKind.PackagedTeamsPermission),
+ CameraSettingsObservationSource.Fixture, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy)
+ : new CameraSettingsObservation(CameraSettingsFinding.PermissionOn, "Fixture: permission on.",
+ Source: CameraSettingsObservationSource.Fixture, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ }
+
+ public Task VerifyTeamsAsync(
+ WindowChoice window, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (!permissionFlow)
+ return Task.FromResult(new CameraVerificationResult(
+ window.Id, CameraVerificationFinding.Ready, true,
+ "Fixture: simulated Teams camera button is on.",
+ IsFixture: true, Reinitialized: true));
+ verificationAttempts++;
+ return Task.FromResult(verificationAttempts == 1
+ ? new CameraVerificationResult(window.Id, CameraVerificationFinding.NeedsReinitialization,
+ false, "Fixture: reopen the camera surface.", IsFixture: true)
+ : new CameraVerificationResult(window.Id, CameraVerificationFinding.Ready,
+ true, "Fixture: simulated verifier passed.", IsFixture: true, Reinitialized: true));
+ }
+
+ public Task ShowTargetAsync(
+ CameraRecoveryTarget target, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return Task.FromResult(new CameraTargetPresentation(true, "Fixture: simulated target shown."));
+ }
+
+ public Task ActivateTargetAsync(
+ CameraRecoveryTarget target, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (target.Kind == CameraRecoveryTargetKind.TeamsCameraButton
+ && target.ObservationId == "fixture-teams-camera-1")
+ {
+ cameraEnabled = true;
+ return Task.FromResult(new CameraTargetControlResult(
+ true, true, "Fixture: simulated Teams camera button invoked."));
+ }
+ if (target.Kind == CameraRecoveryTargetKind.PackagedTeamsPermission
+ && target.AutomationId == CameraRecoveryPinnedTargets.PackagedTeamsCameraToggle)
+ {
+ permissionFlow = true;
+ settingsObservations = Math.Max(settingsObservations, 2);
+ return Task.FromResult(new CameraTargetControlResult(
+ true, true, "Fixture: simulated Teams permission toggle invoked."));
+ }
+ return Task.FromResult(new CameraTargetControlResult(
+ false, true, "Fixture target changed before the approved action."));
+ }
+
+ public Task RestartTeamsAsync(
+ WindowChoice window, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ verificationAttempts = Math.Max(verificationAttempts, 1);
+ return Task.FromResult(new TeamsRestartResult(
+ TeamsRestartFinding.Restarted, "Fixture: simulated Teams restart.", window));
+ }
+}
+
+internal sealed class CameraRecoverySession
+{
+ public CameraRecoveryState State { get; private set; } = CameraRecoveryState.Idle;
+ public CameraRecoveryInteractionMode Mode { get; private set; }
+ public string? TeamsWindowId { get; private set; }
+ public string? TeamsWindowTitle { get; private set; }
+ public CameraRecoveryTarget? Target { get; private set; }
+ public CameraRecoveryTargetKind RecoveryTargetKind { get; private set; }
+ public string Detail { get; private set; } = "";
+ public bool LocalVerifierPassed { get; private set; }
+ public bool TeamsRestartAttempted { get; private set; }
+ public bool CameraOnRequestAuthorized { get; private set; }
+ private bool globalPermissionRecovery;
+ private bool permissionWasRestored;
+
+ public CameraRecoverySession(
+ CameraRecoveryInteractionMode mode = CameraRecoveryInteractionMode.Guide)
+ {
+ Reset(mode);
+ }
+
+ public void Reset(CameraRecoveryInteractionMode mode)
+ {
+ Mode = mode;
+ State = CameraRecoveryState.Idle;
+ TeamsWindowId = null;
+ TeamsWindowTitle = null;
+ Target = null;
+ RecoveryTargetKind = CameraRecoveryTargetKind.Unknown;
+ LocalVerifierPassed = false;
+ TeamsRestartAttempted = false;
+ CameraOnRequestAuthorized = false;
+ globalPermissionRecovery = false;
+ permissionWasRestored = false;
+ Detail = mode == CameraRecoveryInteractionMode.Control
+ ? "Start when you want MSGuide to fix a verified Teams camera control after approval."
+ : "Start when you want guide-only help with a Teams camera.";
+ }
+
+ internal static async Task AssessCurrentAsync(
+ ICameraRecoverySensing sensing, WindowChoice window, CameraRecoveryInteractionMode mode,
+ CancellationToken cancellationToken)
+ {
+ var assessment = new CameraRecoverySession(mode);
+ assessment.Start();
+ assessment.ChooseTeamsWindow(window.Id, window.Title);
+ var observation = await sensing.ObserveTeamsAsync(window, cancellationToken);
+ cancellationToken.ThrowIfCancellationRequested();
+ assessment.ApplyTeamsObservation(observation);
+ if (assessment.CanVerifyTeams)
+ {
+ var verification = await sensing.VerifyTeamsAsync(window, cancellationToken);
+ cancellationToken.ThrowIfCancellationRequested();
+ assessment.ApplyVerification(verification);
+ }
+ return assessment;
+ }
+
+ internal void ApplyReadOnlyAssessment(CameraRecoverySession assessment)
+ {
+ Require(CanStart, "A read-only reassessment must start from idle.");
+ TeamsWindowId = assessment.TeamsWindowId;
+ TeamsWindowTitle = assessment.TeamsWindowTitle;
+ if (assessment.LocalVerifierPassed
+ && assessment.State is CameraRecoveryState.Ready or CameraRecoveryState.FixtureComplete)
+ {
+ LocalVerifierPassed = true;
+ MoveTo(assessment.State, assessment.Detail);
+ }
+ else
+ {
+ MoveTo(CameraRecoveryState.ReadOnlyAssessment,
+ assessment.Detail + " Read-only check complete. Submit your camera request to start a fresh repair.");
+ }
+ }
+
+ public bool CanStart => State == CameraRecoveryState.Idle;
+ public bool CanComplete => LocalVerifierPassed
+ && State is CameraRecoveryState.Ready or CameraRecoveryState.FixtureComplete;
+ public bool CanSelectMode => CanStart || IsTerminal;
+ public bool CanInspectTeams => State == CameraRecoveryState.NeedsTeamsObservation
+ && !string.IsNullOrWhiteSpace(TeamsWindowId);
+ public bool CanOpenSettings => State is CameraRecoveryState.Diagnosis or CameraRecoveryState.NeedsCameraSettings;
+ public bool CanInspectSettings => State == CameraRecoveryState.NeedsSettingsObservation;
+ public bool CanShowTarget => State == CameraRecoveryState.VerifiedTarget && Target is not null;
+ public bool CanCheckChangedSetting => Mode == CameraRecoveryInteractionMode.Guide
+ && State == CameraRecoveryState.VerifiedTarget;
+ public bool CanControlTarget => Mode == CameraRecoveryInteractionMode.Control
+ && State == CameraRecoveryState.VerifiedTarget && Target is not null;
+ public bool CanReturnToTeams => State == CameraRecoveryState.PermissionObservedOn;
+ public bool CanVerifyTeams => State is (CameraRecoveryState.NeedsLocalVerification
+ or CameraRecoveryState.NeedsCameraReinitialization)
+ && !string.IsNullOrWhiteSpace(TeamsWindowId);
+ public bool CanRestartTeams => Mode == CameraRecoveryInteractionMode.Control
+ && State == CameraRecoveryState.NeedsCameraReinitialization
+ && !TeamsRestartAttempted
+ && !string.IsNullOrWhiteSpace(TeamsWindowId);
+ public bool IsTerminal => State is CameraRecoveryState.Ready or CameraRecoveryState.FixtureComplete
+ or CameraRecoveryState.ReadOnlyAssessment
+ or CameraRecoveryState.ControlRevalidationRequired
+ or CameraRecoveryState.WrongSettingsPage or CameraRecoveryState.ManagedOrDisabled
+ or CameraRecoveryState.AlreadyOnOrWrongCause or CameraRecoveryState.StaleOrMoved
+ or CameraRecoveryState.UnresolvedAfterPermission or CameraRecoveryState.Unsupported
+ or CameraRecoveryState.Cancelled;
+
+ public static bool IsCameraHelpIntent(string? prompt)
+ {
+ if (string.IsNullOrWhiteSpace(prompt)) return false;
+ string text = prompt.ToLowerInvariant();
+ bool mentionsCamera = text.Contains("camera") || text.Contains("webcam") || text.Contains("video");
+ bool mentionsTeams = text.Contains("teams") || text.Contains("team's")
+ || text.Contains("team\u2019s") || text.Contains("meeting");
+ bool asksForHelp = text.Contains("help") || text.Contains("fix") || text.Contains("not working")
+ || text.Contains("won't") || text.Contains("cannot") || text.Contains("can't")
+ || text.Contains("permission") || text.Contains("blocked") || text.Contains("off")
+ || text.Contains("check") || text.Contains("working") || text.Contains("status");
+ return mentionsCamera && mentionsTeams && asksForHelp;
+ }
+
+ public void Start(bool authorizeCameraOn = false)
+ {
+ Require(CanStart, "Reset camera recovery before starting another run.");
+ State = CameraRecoveryState.NeedsTeamsObservation;
+ Target = null;
+ RecoveryTargetKind = CameraRecoveryTargetKind.Unknown;
+ LocalVerifierPassed = false;
+ TeamsRestartAttempted = false;
+ CameraOnRequestAuthorized = authorizeCameraOn && Mode == CameraRecoveryInteractionMode.Control;
+ Detail = CameraOnRequestAuthorized
+ ? "Inspecting Teams first. Your Fix request permits one verified camera-on action; permission changes and restart need separate approval."
+ : Mode == CameraRecoveryInteractionMode.Control
+ ? "Open a Teams meeting or prejoin screen. MSGuide will inspect first and ask before changing anything."
+ : "Open a Teams meeting, prejoin, or Settings > Devices screen, then inspect controls only.";
+ }
+
+ public void ChooseTeamsWindow(string windowId, string title)
+ {
+ if (TeamsWindowId is not null && TeamsWindowId != windowId)
+ CameraOnRequestAuthorized = false;
+ bool reinitializing = State is CameraRecoveryState.PermissionObservedOn
+ or CameraRecoveryState.NeedsLocalVerification
+ or CameraRecoveryState.NeedsCameraReinitialization;
+ if (reinitializing)
+ {
+ TeamsWindowId = string.IsNullOrWhiteSpace(windowId) ? null : windowId;
+ TeamsWindowTitle = Clean(title);
+ Target = null;
+ RecoveryTargetKind = CameraRecoveryTargetKind.Unknown;
+ LocalVerifierPassed = false;
+ State = CameraRecoveryState.NeedsCameraReinitialization;
+ Detail = TeamsWindowId is null
+ ? "Choose the reopened Teams window before running the private visual check."
+ : $"Rebound to “{TeamsWindowTitle}”. Open Teams Devices or prejoin, then run Private visual check.";
+ return;
+ }
+ if (IsTerminal) Reset(Mode);
+ if (CanStart) Start();
+ TeamsWindowId = string.IsNullOrWhiteSpace(windowId) ? null : windowId;
+ TeamsWindowTitle = Clean(title);
+ Target = null;
+ LocalVerifierPassed = false;
+ State = CameraRecoveryState.NeedsTeamsObservation;
+ Detail = TeamsWindowId is null
+ ? "Choose the exact Teams window before inspection."
+ : $"Selected “{TeamsWindowTitle}”. Open its meeting, prejoin, or Settings > Devices camera surface, then inspect.";
+ }
+
+ public void ApplyTeamsObservation(TeamsCameraObservation observation)
+ {
+ bool checkingChangedCamera = State == CameraRecoveryState.VerifiedTarget
+ && Target?.Kind == CameraRecoveryTargetKind.TeamsCameraButton;
+ Require(State == CameraRecoveryState.NeedsTeamsObservation || checkingChangedCamera,
+ "Teams observation is not expected now.");
+ if (TeamsWindowId is null || observation.WindowId != TeamsWindowId)
+ {
+ MoveTo(CameraRecoveryState.StaleOrMoved,
+ "The Teams window changed or the observation was for another window. Choose it again.");
+ return;
+ }
+
+ Target = null;
+ switch (observation.Finding)
+ {
+ case TeamsCameraFinding.CameraOff when observation.Target is not null:
+ if (observation.Target.Kind != CameraRecoveryTargetKind.TeamsCameraButton
+ || string.IsNullOrWhiteSpace(observation.Target.ObservationId)
+ || !CameraRecoveryPinnedTargets.IsTeamsTurnCameraOn(observation.Target.Label)
+ && !CameraRecoveryPinnedTargets.IsTeamsCameraToggle(observation.Target.Label))
+ {
+ MoveTo(CameraRecoveryState.Unsupported,
+ "The Teams camera target did not match the pinned camera-on control.");
+ break;
+ }
+ Target = observation.Target;
+ RecoveryTargetKind = CameraRecoveryTargetKind.TeamsCameraButton;
+ MoveTo(CameraRecoveryState.VerifiedTarget,
+ CameraOnRequestAuthorized
+ ? "The verified Teams camera is off. Your submitted Fix request authorizes turning it on once, followed by a local readiness check."
+ : Mode == CameraRecoveryInteractionMode.Control
+ ? "The exact Teams camera control is off. Approve the action to let MSGuide turn it on."
+ : "The exact Teams camera control is off. Choose Show me, turn it on, then check again.");
+ break;
+ case TeamsCameraFinding.CameraOff:
+ MoveTo(CameraRecoveryState.Unsupported,
+ "Teams reports the camera off, but no current verified control was supplied.");
+ break;
+ case TeamsCameraFinding.CameraOn:
+ MoveTo(CameraRecoveryState.NeedsLocalVerification,
+ checkingChangedCamera || Mode == CameraRecoveryInteractionMode.Control
+ ? "The Teams camera control is now on. Running a local readiness check is the next step."
+ : "The Teams camera control is on. Run the private visual check to verify readiness.");
+ break;
+ case TeamsCameraFinding.PermissionMayBeOff:
+ globalPermissionRecovery |= observation.PermissionScope is
+ CameraRecoveryTargetKind.DeviceCameraPermission or CameraRecoveryTargetKind.AppCameraPermission;
+ MoveTo(CameraRecoveryState.Diagnosis,
+ DetailOr(observation.Detail, "Teams indicates camera permission may be blocking access."));
+ break;
+ case TeamsCameraFinding.PermissionAlreadyOnOrDifferentCause:
+ if (globalPermissionRecovery || permissionWasRestored)
+ {
+ MoveTo(CameraRecoveryState.NeedsLocalVerification,
+ "Camera permissions have been restored. Verify the Teams Devices preview locally; permission-on alone is not readiness.");
+ break;
+ }
+ MoveTo(CameraRecoveryState.AlreadyOnOrWrongCause,
+ "Permission does not appear to be the cause. Do not force the permission path.");
+ break;
+ case TeamsCameraFinding.ManagedOrDisabled:
+ MoveTo(CameraRecoveryState.ManagedOrDisabled,
+ DetailOr(observation.Detail, "Camera access appears disabled or managed. MSGuide will not change policy."));
+ break;
+ case TeamsCameraFinding.StaleOrMoved:
+ MoveTo(CameraRecoveryState.StaleOrMoved,
+ "Teams moved, closed, or changed during observation.");
+ break;
+ default:
+ MoveTo(CameraRecoveryState.Unsupported,
+ DetailOr(observation.Detail,
+ "Controls-only Teams sensing is unavailable or unsupported. No camera-recovery screenshot was captured."));
+ break;
+ }
+ }
+
+ public void PrepareToOpenSettings()
+ {
+ Require(State is CameraRecoveryState.Diagnosis or CameraRecoveryState.NeedsCameraSettings,
+ "Camera Settings is not the next verified step.");
+ MoveTo(CameraRecoveryState.NeedsCameraSettings,
+ "Open Windows Camera privacy settings. You remain responsible for every setting change.");
+ }
+
+ public void MarkSettingsOpened()
+ {
+ Require(State == CameraRecoveryState.NeedsCameraSettings, "Camera Settings was not expected now.");
+ MoveTo(CameraRecoveryState.NeedsSettingsObservation,
+ "Windows Settings launch requested. Leave it visible and verify the Camera privacy page before trusting any toggle.");
+ }
+
+ public void ApplySettingsObservation(CameraSettingsObservation observation)
+ {
+ Require(State == CameraRecoveryState.NeedsSettingsObservation
+ || State == CameraRecoveryState.VerifiedTarget
+ && CameraRecoveryPinnedTargets.IsPermission(RecoveryTargetKind),
+ "A Camera Settings observation is not expected now.");
+ bool initialSettingsObservation = State == CameraRecoveryState.NeedsSettingsObservation;
+ Target = null;
+ if (observation.Finding is CameraSettingsFinding.PermissionOff or CameraSettingsFinding.PermissionOn
+ or CameraSettingsFinding.ManagedOrDisabled
+ && observation.Page != CameraSettingsPage.CameraPrivacy)
+ {
+ MoveTo(CameraRecoveryState.WrongSettingsPage,
+ "Windows Settings did not verify as the Camera privacy page. No permission claim or target was accepted.");
+ return;
+ }
+ if (observation.Finding is CameraSettingsFinding.PermissionOff or CameraSettingsFinding.PermissionOn
+ or CameraSettingsFinding.ManagedOrDisabled
+ && (!observation.ProbeValidated || observation.Source == CameraSettingsObservationSource.Unknown))
+ {
+ MoveTo(CameraRecoveryState.Unsupported,
+ "Camera Settings sensing lacked a disclosed, successfully probed method. No target or permission claim was accepted.");
+ return;
+ }
+ switch (observation.Finding)
+ {
+ case CameraSettingsFinding.WrongPage:
+ MoveTo(CameraRecoveryState.WrongSettingsPage,
+ "Windows Settings did not verify as the Camera privacy page. No permission claim or target was accepted.");
+ break;
+ case CameraSettingsFinding.PermissionOff when observation.Target is not null:
+ if (!CameraRecoveryPinnedTargets.IsPermissionTarget(
+ observation.Target.AutomationId, observation.Target.Kind)
+ || string.IsNullOrWhiteSpace(observation.Target.ObservationId))
+ {
+ MoveTo(CameraRecoveryState.Unsupported,
+ "The Camera Settings target did not match its verified permission scope.");
+ break;
+ }
+ Target = observation.Target;
+ RecoveryTargetKind = observation.Target.Kind;
+ globalPermissionRecovery |= RecoveryTargetKind is
+ CameraRecoveryTargetKind.DeviceCameraPermission or CameraRecoveryTargetKind.AppCameraPermission;
+ MoveTo(CameraRecoveryState.VerifiedTarget,
+ RecoveryTargetKind != CameraRecoveryTargetKind.PackagedTeamsPermission
+ ? CameraPermissionEvidence.DescribeBlock(RecoveryTargetKind)
+ + (Mode == CameraRecoveryInteractionMode.Control
+ ? " Review this scope, then approve this setting only."
+ : " Choose Show me and change this setting yourself, then check again.")
+ : Mode == CameraRecoveryInteractionMode.Control
+ ? "The exact packaged Teams permission is off. Approve the action to let MSGuide turn on only this toggle."
+ : "A current Camera Settings target was verified. Choose Show me; MSGuide will not click it.");
+ break;
+ case CameraSettingsFinding.PermissionOff:
+ MoveTo(CameraRecoveryState.Unsupported,
+ "Camera permission appears off, but no current verified target was supplied. No highlight or click was attempted.");
+ break;
+ case CameraSettingsFinding.PermissionOn:
+ permissionWasRestored = !initialSettingsObservation || globalPermissionRecovery
+ || CameraOnRequestAuthorized;
+ MoveTo(initialSettingsObservation && !globalPermissionRecovery && !CameraOnRequestAuthorized
+ ? CameraRecoveryState.AlreadyOnOrWrongCause
+ : CameraRecoveryState.PermissionObservedOn,
+ globalPermissionRecovery
+ ? CameraOnRequestAuthorized
+ ? "Windows camera permissions are on. Continuing with fresh Teams inspection under your camera repair request."
+ : "Windows camera permissions are now on. Return to Teams for fresh inspection; the Teams camera action needs its own approval."
+ : initialSettingsObservation
+ ? "Camera permission was already on before any guided change. Do not force the permission path."
+ : Mode == CameraRecoveryInteractionMode.Control
+ ? "Camera permission is observed on after the approved action. This alone does not prove the Teams camera is ready."
+ : "Camera permission is observed on after the user's change. This alone does not prove the Teams camera is ready.");
+ break;
+ case CameraSettingsFinding.ManagedOrDisabled:
+ MoveTo(CameraRecoveryState.ManagedOrDisabled,
+ DetailOr(observation.Detail, "The camera setting appears disabled or managed. MSGuide will not override policy."));
+ break;
+ case CameraSettingsFinding.StaleOrMoved:
+ MoveTo(CameraRecoveryState.StaleOrMoved,
+ "Camera Settings moved, closed, or changed during observation.");
+ break;
+ default:
+ MoveTo(CameraRecoveryState.Unsupported,
+ DetailOr(observation.Detail,
+ "Controls-only Camera Settings sensing is unavailable or unsupported. No camera-recovery screenshot was captured."));
+ break;
+ }
+ }
+
+ public void RecordTargetPresentation(CameraTargetPresentation presentation)
+ {
+ Require(State == CameraRecoveryState.VerifiedTarget, "There is no current verified target to show.");
+ Detail = presentation.Shown
+ ? Mode == CameraRecoveryInteractionMode.Control
+ ? "Verified target shown. Review it, then approve the single action when ready."
+ : "Verified target shown. Make the change yourself, then choose I changed it - check."
+ : "The verified target could not be shown. No click was attempted.";
+ if (RecoveryTargetKind is CameraRecoveryTargetKind.DeviceCameraPermission
+ or CameraRecoveryTargetKind.AppCameraPermission)
+ Detail = CameraPermissionEvidence.DescribeBlock(RecoveryTargetKind) + " " + Detail;
+ }
+
+ public void RecordControlFailure(string detail) =>
+ MoveTo(CameraRecoveryState.Unsupported,
+ DetailOr(detail, "The approved action could not be completed safely. No retry occurred."));
+
+ internal void RequireControlRevalidation(string detail)
+ {
+ Target = null;
+ MoveTo(CameraRecoveryState.ControlRevalidationRequired,
+ DetailOr(detail, "The camera control could not be revalidated. No action was started."));
+ }
+
+ public void MarkReturnedToTeams()
+ {
+ Require(State == CameraRecoveryState.PermissionObservedOn, "Returning to Teams is not the next step.");
+ if (globalPermissionRecovery || CameraOnRequestAuthorized)
+ {
+ MoveTo(CameraRecoveryState.NeedsTeamsObservation,
+ CameraOnRequestAuthorized
+ ? "Camera permissions are on. Inspecting Teams again before the camera-on action authorized by your request."
+ : "Camera permissions are on. Inspect Teams again; turning on its camera requires a separate approval.");
+ return;
+ }
+ MoveTo(CameraRecoveryState.NeedsLocalVerification,
+ "Back in Teams, run the private visual check. Permission-on alone is not camera-ready.");
+ }
+
+ public void ApplyVerification(CameraVerificationResult result)
+ {
+ Require(State is CameraRecoveryState.NeedsLocalVerification or CameraRecoveryState.NeedsCameraReinitialization,
+ "A Teams camera verification is not expected now.");
+ if (TeamsWindowId is null || result.WindowId != TeamsWindowId
+ || result.Finding == CameraVerificationFinding.StaleOrMoved)
+ {
+ MoveTo(CameraRecoveryState.StaleOrMoved,
+ "Teams moved, closed, or changed before local verification.");
+ return;
+ }
+
+ LocalVerifierPassed = result.LocalVerifierPassed;
+ if (result.Finding == CameraVerificationFinding.Ready && !result.Reinitialized)
+ {
+ LocalVerifierPassed = false;
+ MoveTo(CameraRecoveryState.NeedsCameraReinitialization,
+ RecoveryTargetKind == CameraRecoveryTargetKind.PackagedTeamsPermission
+ ? "The verifier did not prove that Teams reinitialized its camera after permission restoration. Reopen prejoin or Teams Devices, or relaunch Teams, then check again."
+ : "The verifier did not prove that Teams started the camera. Reopen prejoin or relaunch Teams, then check again.");
+ return;
+ }
+ if (result.Finding == CameraVerificationFinding.NeedsReinitialization)
+ {
+ LocalVerifierPassed = false;
+ MoveTo(CameraRecoveryState.NeedsCameraReinitialization,
+ RecoveryTargetKind == CameraRecoveryTargetKind.PackagedTeamsPermission
+ ? "The existing Teams camera session may have survived the permission change. Reopen prejoin or the camera surface, or approve a Teams restart."
+ : "Teams reports the camera control on, but active camera use was not verified. Reopen the camera surface or approve a Teams restart.");
+ return;
+ }
+ if (result.Finding is CameraVerificationFinding.Ready or CameraVerificationFinding.AlreadyReady
+ && result.LocalVerifierPassed)
+ {
+ MoveTo(result.IsFixture ? CameraRecoveryState.FixtureComplete : CameraRecoveryState.Ready,
+ result.IsFixture
+ ? "Fixture complete: the simulated verifier passed. Real camera-ready was not claimed."
+ : result.Finding == CameraVerificationFinding.AlreadyReady
+ ? "Already working: Teams reports camera on and Windows reports active camera use. No change was needed."
+ : "Locally verified: the supplied Teams camera readiness check passed.");
+ return;
+ }
+
+ LocalVerifierPassed = false;
+ MoveTo(result.Finding == CameraVerificationFinding.Unsupported
+ ? CameraRecoveryState.Unsupported
+ : CameraRecoveryState.UnresolvedAfterPermission,
+ result.Finding is CameraVerificationFinding.Ready or CameraVerificationFinding.AlreadyReady
+ ? "The verifier did not pass. Camera-ready was not claimed."
+ : result.Finding == CameraVerificationFinding.Unsupported
+ ? "The connected local Teams verifier does not support this state. Camera-ready was not claimed."
+ : "The camera control is on, but Teams is still not locally verified ready.");
+ }
+
+ public void RecordTeamsRestart(TeamsRestartResult result)
+ {
+ Require(CanRestartTeams, "A Teams restart is not currently approved.");
+ TeamsRestartAttempted = true;
+ if (result.Finding == TeamsRestartFinding.Restarted)
+ {
+ Detail = result.ReopenedWindow is null
+ ? "Teams restart was requested. Choose the reopened Teams window, open its camera surface, then verify again."
+ : "Teams restarted after separate approval. Verifying the reopened camera surface is the next step.";
+ return;
+ }
+ MoveTo(result.Finding == TeamsRestartFinding.StaleOrMoved
+ ? CameraRecoveryState.StaleOrMoved
+ : CameraRecoveryState.Unsupported,
+ DetailOr(result.Detail, "Teams could not be restarted safely."));
+ }
+
+ public void MarkUnsupported(string detail) =>
+ MoveTo(CameraRecoveryState.Unsupported, DetailOr(detail, "Camera recovery is unsupported."));
+
+ public void MarkStale(string detail) =>
+ MoveTo(CameraRecoveryState.StaleOrMoved, DetailOr(detail, "The observed screen is no longer current."));
+
+ public void Cancel(string detail = "Stopped. MSGuide will perform no further camera actions.")
+ {
+ Target = null;
+ LocalVerifierPassed = false;
+ MoveTo(CameraRecoveryState.Cancelled, detail);
+ }
+
+ internal bool ConsumeCameraOnRequest()
+ {
+ if (!CameraOnRequestAuthorized || !CanControlTarget || TeamsWindowId is null
+ || Target?.Kind != CameraRecoveryTargetKind.TeamsCameraButton) return false;
+ CameraOnRequestAuthorized = false;
+ return true;
+ }
+
+ private void MoveTo(CameraRecoveryState state, string detail)
+ {
+ State = state;
+ Detail = Clean(detail);
+ if (IsTerminal) CameraOnRequestAuthorized = false;
+ }
+
+ private static string DetailOr(string detail, string fallback) =>
+ string.IsNullOrWhiteSpace(detail) ? fallback : Clean(detail);
+
+ private static string Clean(string? value)
+ {
+ string clean = (value ?? "").Trim();
+ return clean.Length <= 500 ? clean : clean[..500];
+ }
+
+ private static void Require(bool condition, string message)
+ {
+ if (!condition) throw new InvalidOperationException(message);
+ }
+}
diff --git a/desktop/CameraRecoveryTests.cs b/desktop/CameraRecoveryTests.cs
new file mode 100644
index 0000000..b4bd82c
--- /dev/null
+++ b/desktop/CameraRecoveryTests.cs
@@ -0,0 +1,522 @@
+namespace MSGuide.Desktop;
+
+internal static class CameraRecoveryTests
+{
+ public static void Run()
+ {
+ static void Check(bool condition, string name)
+ {
+ if (!condition) throw new InvalidOperationException($"Camera recovery test failed: {name}.");
+ }
+
+ RunPermissionChecks();
+ RunCurrentStateChecks();
+ Check(CameraRecoverySession.IsCameraHelpIntent("Help me fix my camera in Teams"), "typed intent");
+ Check(CameraRecoverySession.IsCameraHelpIntent("Is my camera working in Teams?"), "camera status question");
+ Check(CameraRecoverySession.IsCameraHelpIntent("Check my team's camera."),
+ "natural possessive speech transcript routes to camera help without changing the words");
+ Check(CameraRecoverySession.IsCameraHelpIntent("My meeting video is not working"), "meeting video intent");
+ Check(!CameraRecoverySession.IsCameraHelpIntent("Help me find the build error"), "unrelated intent");
+ ElementInfo[] partialControls =
+ [
+ new("group", "Video settings", [0.1, 0.1, 0.2, 0.2],
+ AutomationId: CameraRecoveryPinnedTargets.TeamsVideoSettings),
+ new("button", "Turn camera on (Ctrl+Shift+O)", [0.4, 0.1, 0.2, 0.2],
+ ToggleState: "off")
+ ];
+ Check(new AutomationReadResult(partialControls, "", "", true).RequireComplete() == partialControls,
+ "complete camera controls remain usable");
+ bool incompleteRejected = false;
+ try { new AutomationReadResult(partialControls, "", "", false).RequireComplete(); }
+ catch (IncompleteAutomationReadException) { incompleteRejected = true; }
+ Check(incompleteRejected,
+ "incomplete controls cannot authorize targets or permission fallback");
+ Check(CameraRecoveryPinnedTargets.IsTeamsTurnCameraOn(
+ "Turn camera on (Ctrl+Shift+O)")
+ && CameraRecoveryPinnedTargets.IsTeamsTurnCameraOff(
+ "Turn camera off (Ctrl+Shift+O)")
+ && !CameraRecoveryPinnedTargets.IsTeamsTurnCameraOn(
+ "Turn camera on and share the screen")
+ && !CameraRecoveryPinnedTargets.IsTeamsTurnCameraOn(
+ "Turn camera on (for everyone)"),
+ "Teams camera shortcut suffix normalization");
+
+ var pendingSensing = new PendingCameraRecoverySensing();
+ var pendingResult = pendingSensing.ObserveTeamsAsync(
+ new WindowChoice(0, 0, "Synthetic Teams"), CancellationToken.None).GetAwaiter().GetResult();
+ Check(pendingSensing.Mode == CameraRecoverySensingMode.UnsupportedFallback
+ && pendingResult.Finding == TeamsCameraFinding.Unsupported
+ && pendingResult.Detail.Contains("did not take a screenshot", StringComparison.Ordinal),
+ "explicit unsupported fallback");
+
+ var cameraControlSession =
+ new CameraRecoverySession(CameraRecoveryInteractionMode.Control);
+ cameraControlSession.Start();
+ cameraControlSession.ChooseTeamsWindow("teams-camera", "Meeting | Microsoft Teams");
+ cameraControlSession.ApplyTeamsObservation(new(
+ "teams-camera", TeamsCameraFinding.CameraOff, "",
+ new CameraRecoveryTarget(
+ "teams-camera-target", "Turn camera on (Ctrl+Shift+O)",
+ "camera-button", CameraRecoveryTargetKind.TeamsCameraButton)));
+ Check(cameraControlSession.State == CameraRecoveryState.VerifiedTarget
+ && cameraControlSession.CanControlTarget
+ && !cameraControlSession.CanCheckChangedSetting,
+ "control mode gates exact Teams camera action");
+ cameraControlSession.ApplyTeamsObservation(new(
+ "teams-camera", TeamsCameraFinding.CameraOn, ""));
+ Check(cameraControlSession.State == CameraRecoveryState.NeedsLocalVerification,
+ "camera-on observation requires readiness verification");
+ cameraControlSession.ApplyVerification(new(
+ "teams-camera", CameraVerificationFinding.NeedsReinitialization, false, ""));
+ Check(cameraControlSession.CanRestartTeams, "restart requires separate approval state");
+ cameraControlSession.RecordTeamsRestart(new(
+ TeamsRestartFinding.Restarted, "", new WindowChoice(0, 0, "Microsoft Teams")));
+ Check(!cameraControlSession.CanRestartTeams
+ && cameraControlSession.TeamsRestartAttempted,
+ "restart approval is single-use");
+
+ var session = new CameraRecoverySession();
+ Check(session.State == CameraRecoveryState.Idle && !session.LocalVerifierPassed
+ && session.CanStart && session.CanSelectMode, "idle mode selection");
+ session.Start();
+ Check(!session.CanStart && !session.CanSelectMode, "active recovery locks mode selection");
+ session.ChooseTeamsWindow("teams-1", "Weekly meeting | Microsoft Teams");
+ Check(session.State == CameraRecoveryState.NeedsTeamsObservation && session.CanInspectTeams, "choose Teams");
+ session.ApplyTeamsObservation(new("teams-1", TeamsCameraFinding.PermissionMayBeOff, ""));
+ Check(session.State == CameraRecoveryState.Diagnosis && session.CanOpenSettings, "diagnosis");
+ session.PrepareToOpenSettings();
+ session.MarkSettingsOpened();
+ Check(session.State == CameraRecoveryState.NeedsSettingsObservation && session.CanInspectSettings, "settings opened");
+ session.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOff, "",
+ new CameraRecoveryTarget("settings-observation-1", "Microsoft Teams Currently in use",
+ CameraRecoveryPinnedTargets.PackagedTeamsCameraToggle,
+ CameraRecoveryTargetKind.PackagedTeamsPermission),
+ CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ Check(session.State == CameraRecoveryState.VerifiedTarget && session.CanShowTarget
+ && session.CanCheckChangedSetting
+ && !session.CanControlTarget
+ && session.Target?.AutomationId == CameraRecoveryPinnedTargets.PackagedTeamsCameraToggle,
+ "verified target");
+ session.RecordTargetPresentation(new(true, ""));
+ session.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOn, "",
+ Source: CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ Check(session.State == CameraRecoveryState.PermissionObservedOn && !session.LocalVerifierPassed
+ && session.CanReturnToTeams, "permission alone not ready");
+ session.MarkReturnedToTeams();
+ session.ApplyVerification(new("teams-1", CameraVerificationFinding.Ready, true, ""));
+ Check(session.State == CameraRecoveryState.NeedsCameraReinitialization
+ && !session.LocalVerifierPassed, "first verification requires camera reinitialization");
+ session.ApplyVerification(new("teams-1", CameraVerificationFinding.Ready, true, "",
+ Reinitialized: true));
+ Check(session.State == CameraRecoveryState.Ready && session.LocalVerifierPassed
+ && session.CanSelectMode, "local verifier ready");
+
+ var falseReady = PermissionOnSession();
+ falseReady.ApplyVerification(new("teams-1", CameraVerificationFinding.NeedsReinitialization,
+ false, ""));
+ falseReady.ApplyVerification(new("teams-1", CameraVerificationFinding.Ready, false, "",
+ Reinitialized: true));
+ Check(falseReady.State == CameraRecoveryState.UnresolvedAfterPermission
+ && !falseReady.LocalVerifierPassed, "false verifier cannot claim ready");
+
+ var unresolved = PermissionOnSession();
+ unresolved.ApplyVerification(new("teams-1", CameraVerificationFinding.Unresolved, false, ""));
+ Check(unresolved.State == CameraRecoveryState.UnresolvedAfterPermission, "unresolved after permission");
+
+ var reinitialize = PermissionOnSession();
+ reinitialize.ApplyVerification(new("teams-1",
+ CameraVerificationFinding.NeedsReinitialization, false, ""));
+ Check(reinitialize.State == CameraRecoveryState.NeedsCameraReinitialization
+ && reinitialize.CanVerifyTeams && !reinitialize.LocalVerifierPassed,
+ "live camera session requires reinitialization");
+ reinitialize.ApplyVerification(new("teams-1", CameraVerificationFinding.Ready, true, "",
+ Reinitialized: true));
+ Check(reinitialize.State == CameraRecoveryState.Ready, "ready after explicit reinitialization");
+
+ var alreadyOn = StartedSession();
+ alreadyOn.ApplyTeamsObservation(new("teams-1",
+ TeamsCameraFinding.PermissionAlreadyOnOrDifferentCause, "Camera ready"));
+ Check(alreadyOn.State == CameraRecoveryState.AlreadyOnOrWrongCause
+ && !alreadyOn.Detail.Contains("Camera ready", StringComparison.OrdinalIgnoreCase),
+ "already on wrong cause cannot inject ready claim");
+
+ var managed = StartedSession();
+ managed.ApplyTeamsObservation(new("teams-1", TeamsCameraFinding.ManagedOrDisabled, ""));
+ Check(managed.State == CameraRecoveryState.ManagedOrDisabled, "managed");
+
+ var stale = StartedSession();
+ stale.ApplyTeamsObservation(new("other-window", TeamsCameraFinding.PermissionMayBeOff, ""));
+ Check(stale.State == CameraRecoveryState.StaleOrMoved, "stale target");
+
+ var unsupported = StartedSession();
+ unsupported.ApplyTeamsObservation(new("teams-1", TeamsCameraFinding.Unsupported, ""));
+ Check(unsupported.State == CameraRecoveryState.Unsupported
+ && unsupported.Detail.Contains("No camera-recovery screenshot", StringComparison.Ordinal),
+ "unsupported");
+ var unsupportedDetail = StartedSession();
+ unsupportedDetail.ApplyTeamsObservation(new(
+ "teams-1", TeamsCameraFinding.Unsupported,
+ "Open the selected Teams prejoin before inspecting again."));
+ Check(unsupportedDetail.Detail == "Open the selected Teams prejoin before inspecting again.",
+ "unsupported observation detail remains visible");
+
+ var settingsManaged = SettingsSession();
+ settingsManaged.ApplySettingsObservation(new(CameraSettingsFinding.ManagedOrDisabled, "",
+ Source: CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ Check(settingsManaged.State == CameraRecoveryState.ManagedOrDisabled, "settings managed");
+
+ var settingsStale = SettingsSession();
+ settingsStale.ApplySettingsObservation(new(CameraSettingsFinding.StaleOrMoved, ""));
+ Check(settingsStale.State == CameraRecoveryState.StaleOrMoved, "settings stale");
+
+ var unprovenSettings = SettingsSession();
+ unprovenSettings.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOn, "",
+ Source: CameraSettingsObservationSource.ControlsOnly, ProbeValidated: false,
+ Page: CameraSettingsPage.CameraPrivacy));
+ Check(unprovenSettings.State == CameraRecoveryState.Unsupported
+ && unprovenSettings.Detail.Contains("successfully probed method", StringComparison.Ordinal),
+ "unproven Settings UIA rejected");
+
+ var wrongSettingsPage = SettingsSession();
+ wrongSettingsPage.ApplySettingsObservation(new(CameraSettingsFinding.WrongPage, "",
+ Source: CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.Other));
+ Check(wrongSettingsPage.State == CameraRecoveryState.WrongSettingsPage
+ && !wrongSettingsPage.LocalVerifierPassed, "Settings URI landing verified");
+
+ var wrongPinnedTarget = SettingsSession();
+ wrongPinnedTarget.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOff, "",
+ new CameraRecoveryTarget("wrong", "Another app", "wrong-toggle"),
+ CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ Check(wrongPinnedTarget.State == CameraRecoveryState.Unsupported,
+ "wrong pinned target rejected");
+
+ var alreadyOnInSettings = SettingsSession();
+ alreadyOnInSettings.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOn, "",
+ Source: CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ Check(alreadyOnInSettings.State == CameraRecoveryState.AlreadyOnOrWrongCause
+ && !alreadyOnInSettings.CanReturnToTeams, "initial Settings permission already on");
+
+ var cancelled = StartedSession();
+ cancelled.Cancel();
+ Check(cancelled.State == CameraRecoveryState.Cancelled && !cancelled.LocalVerifierPassed
+ && cancelled.CanSelectMode, "cancelled mode selection");
+ cancelled.ChooseTeamsWindow("teams-2", "New Microsoft Teams window");
+ Check(cancelled.State == CameraRecoveryState.NeedsTeamsObservation
+ && cancelled.TeamsWindowId == "teams-2" && !cancelled.CanSelectMode,
+ "terminal window selection resets before restarting");
+
+ var rebind = PermissionOnSession();
+ rebind.ApplyVerification(new("teams-1", CameraVerificationFinding.NeedsReinitialization,
+ false, ""));
+ rebind.ChooseTeamsWindow("teams-2", "Reopened Microsoft Teams");
+ Check(rebind.State == CameraRecoveryState.NeedsCameraReinitialization
+ && rebind.TeamsWindowId == "teams-2" && rebind.CanVerifyTeams,
+ "reopened Teams window can be rebound without losing permission evidence");
+
+ bool invalidTransitionRejected = false;
+ try { new CameraRecoverySession().MarkReturnedToTeams(); }
+ catch (InvalidOperationException) { invalidTransitionRejected = true; }
+ Check(invalidTransitionRejected, "invalid transition");
+
+ var fixture = new FixtureCameraRecoverySensing();
+ var fixtureWindow = new WindowChoice(0, 0, "Synthetic Teams fixture");
+ var fixtureSession =
+ new CameraRecoverySession(CameraRecoveryInteractionMode.Control);
+ fixtureSession.Start();
+ fixtureSession.ChooseTeamsWindow(fixtureWindow.Id, fixtureWindow.Title);
+ fixtureSession.ApplyTeamsObservation(
+ fixture.ObserveTeamsAsync(fixtureWindow, CancellationToken.None).GetAwaiter().GetResult());
+ Check(fixtureSession.CanControlTarget
+ && fixtureSession.Target?.Kind == CameraRecoveryTargetKind.TeamsCameraButton,
+ "fixture starts with Teams camera off");
+ Check(fixture.ActivateTargetAsync(
+ fixtureSession.Target!, CancellationToken.None).GetAwaiter().GetResult().Invoked,
+ "fixture control action");
+ fixtureSession.ApplyTeamsObservation(
+ fixture.ObserveTeamsAsync(fixtureWindow, CancellationToken.None).GetAwaiter().GetResult());
+ fixtureSession.ApplyVerification(
+ fixture.VerifyTeamsAsync(fixtureWindow, CancellationToken.None).GetAwaiter().GetResult());
+ Check(fixture.Mode == CameraRecoverySensingMode.Fixture
+ && fixtureSession.State == CameraRecoveryState.FixtureComplete
+ && fixtureSession.CanSelectMode
+ && fixtureSession.Detail.Contains("not claimed", StringComparison.OrdinalIgnoreCase),
+ "deterministic camera-button fixture path");
+ fixtureSession.Reset(CameraRecoveryInteractionMode.Guide);
+ Check(fixtureSession.State == CameraRecoveryState.Idle
+ && fixtureSession.Mode == CameraRecoveryInteractionMode.Guide
+ && fixtureSession.CanStart && fixtureSession.CanSelectMode
+ && fixtureSession.TeamsWindowId is null && fixtureSession.Target is null
+ && !fixtureSession.LocalVerifierPassed && !fixtureSession.TeamsRestartAttempted,
+ "start over returns to editable mode state");
+
+ fixture.Reset();
+ var permissionFixtureSession = new CameraRecoverySession();
+ permissionFixtureSession.Start();
+ permissionFixtureSession.ChooseTeamsWindow(fixtureWindow.Id, fixtureWindow.Title);
+ permissionFixtureSession.ApplyTeamsObservation(new(
+ fixtureWindow.Id, TeamsCameraFinding.PermissionMayBeOff, ""));
+ permissionFixtureSession.PrepareToOpenSettings();
+ permissionFixtureSession.MarkSettingsOpened();
+ permissionFixtureSession.ApplySettingsObservation(
+ fixture.ObserveSettingsAsync(CancellationToken.None).GetAwaiter().GetResult());
+ permissionFixtureSession.ApplySettingsObservation(
+ fixture.ObserveSettingsAsync(CancellationToken.None).GetAwaiter().GetResult());
+ permissionFixtureSession.MarkReturnedToTeams();
+ permissionFixtureSession.ApplyVerification(
+ fixture.VerifyTeamsAsync(fixtureWindow, CancellationToken.None).GetAwaiter().GetResult());
+ Check(permissionFixtureSession.State == CameraRecoveryState.NeedsCameraReinitialization,
+ "permission fixture still requires camera reinitialization");
+
+ fixture.Reset();
+ Check(fixture.ObserveTeamsAsync(
+ fixtureWindow, CancellationToken.None).GetAwaiter().GetResult().Finding
+ == TeamsCameraFinding.CameraOff, "fixture restarts from Teams camera off");
+ }
+
+ private static void RunCurrentStateChecks()
+ {
+ static void Check(bool condition, string name)
+ {
+ if (!condition) throw new InvalidOperationException($"Current camera state test failed: {name}.");
+ }
+ var now = DateTimeOffset.UtcNow;
+ var already = LiveCameraRecoverySensing.MatchActiveCameraUse("teams-1", null, now.AddMinutes(-5));
+ Check(already is { Finding: CameraVerificationFinding.AlreadyReady, LocalVerifierPassed: true, Reinitialized: false },
+ "current active use does not require this session to turn the camera on");
+ Check(LiveCameraRecoverySensing.MatchActiveCameraUse("teams-1", null, null) is null,
+ "camera-on without active-use evidence is not ready");
+ Check(LiveCameraRecoverySensing.MatchActiveCameraUse("teams-1", now, now.AddMinutes(-5)) is null,
+ "a new repair still requires camera use after that repair");
+ Check(LiveCameraRecoverySensing.MatchActiveCameraUse("teams-1", now, now)
+ is { Finding: CameraVerificationFinding.Ready, Reinitialized: true },
+ "fresh repaired camera use remains supported");
+
+ var falseReady = StartedSession();
+ falseReady.ApplyTeamsObservation(new("teams-1", TeamsCameraFinding.CameraOn, ""));
+ falseReady.ApplyVerification(already! with { LocalVerifierPassed = false });
+ Check(!falseReady.LocalVerifierPassed && falseReady.State != CameraRecoveryState.Ready,
+ "already-ready still requires a passed current verifier");
+
+ var window = new WindowChoice(0, 0, "Synthetic current Teams camera");
+ foreach (var mode in new[] { CameraRecoveryInteractionMode.Guide, CameraRecoveryInteractionMode.Control })
+ {
+ var sensing = new CurrentStateSensing();
+ var session = new CameraRecoverySession(mode);
+ for (int round = 0; round < 2; round++)
+ {
+ session.Reset(mode);
+ var assessment = CameraRecoverySession.AssessCurrentAsync(
+ sensing, window, mode, CancellationToken.None).GetAwaiter().GetResult();
+ session.ApplyReadOnlyAssessment(assessment);
+ Check(session.State == CameraRecoveryState.Ready && session.CanSelectMode
+ && !session.CanControlTarget && !session.CanRestartTeams
+ && session.Detail.StartsWith("Already working:", StringComparison.Ordinal),
+ "start over recognizes current readiness without authorizing a change");
+ }
+ Check(sensing.Observations == 2 && sensing.Verifications == 2,
+ "every reassessment obtains fresh observations and verification");
+ sensing.CameraOn = false;
+ session.Reset(mode);
+ session.ApplyReadOnlyAssessment(CameraRecoverySession.AssessCurrentAsync(
+ sensing, window, mode, CancellationToken.None).GetAwaiter().GetResult());
+ Check(session.State == CameraRecoveryState.ReadOnlyAssessment && session.CanSelectMode
+ && session.Target is null && !session.CanControlTarget && !session.CanShowTarget
+ && !session.LocalVerifierPassed,
+ "an unresolved reassessment keeps a visible terminal result, editable modes, and no action authority");
+ }
+ }
+
+ private sealed class CurrentStateSensing : ICameraRecoverySensing
+ {
+ public CameraRecoverySensingMode Mode => CameraRecoverySensingMode.Connected;
+ public bool CameraOn { get; set; } = true;
+ public int Observations { get; private set; }
+ public int Verifications { get; private set; }
+ public void Reset() { }
+
+ public Task ObserveTeamsAsync(WindowChoice window, CancellationToken token)
+ {
+ token.ThrowIfCancellationRequested();
+ Observations++;
+ return Task.FromResult(CameraOn
+ ? new TeamsCameraObservation(window.Id, TeamsCameraFinding.CameraOn, "")
+ : new TeamsCameraObservation(window.Id, TeamsCameraFinding.CameraOff, "",
+ new("current-camera", "Turn camera on", "camera", CameraRecoveryTargetKind.TeamsCameraButton)));
+ }
+
+ public Task VerifyTeamsAsync(WindowChoice window, CancellationToken token)
+ {
+ token.ThrowIfCancellationRequested();
+ Verifications++;
+ return Task.FromResult(new CameraVerificationResult(
+ window.Id, CameraVerificationFinding.AlreadyReady, true, ""));
+ }
+
+ public Task ObserveSettingsAsync(CancellationToken token) =>
+ throw new InvalidOperationException("Current camera reassessment must not navigate to Settings.");
+
+ public Task ShowTargetAsync(CameraRecoveryTarget target, CancellationToken token) =>
+ throw new InvalidOperationException("Current camera reassessment must not present an actionable target.");
+ }
+
+ private static void RunPermissionChecks()
+ {
+ static void Check(bool condition, string name)
+ {
+ if (!condition) throw new InvalidOperationException($"Camera permission test failed: {name}.");
+ }
+
+ var deviceBlock = CameraPermissionEvidence.FromConsent(false, "Deny", "Allow", "Allow");
+ Check(deviceBlock.State == CameraPermissionState.Off
+ && deviceBlock.Scope == CameraRecoveryTargetKind.DeviceCameraPermission,
+ "device-wide denial overrides Teams Allow");
+ Check(CameraPermissionEvidence.FromConsent(true, "Allow", "Allow", "Allow").State
+ == CameraPermissionState.Managed,
+ "policy denial is never offered as a fix");
+ Check(CameraPermissionEvidence.FromConsent(false, null, "Allow", "Allow").State
+ == CameraPermissionState.Unknown,
+ "missing device evidence is not permission-on");
+ Check(CameraPermissionEvidence.FromConsent(false, "Allow", "Deny", "Allow").Scope
+ == CameraRecoveryTargetKind.AppCameraPermission,
+ "app-level denial overrides Teams Allow");
+ Check(CameraPermissionEvidence.FromConsent(false, "Allow", "Allow", "Allow").State
+ == CameraPermissionState.On,
+ "all permission levels must allow access");
+
+ var device = new ElementInfo("button", "Camera access", [0.1, 0.1, 0.2, 0.1],
+ TargetId: "device-off", AutomationId: CameraRecoveryPinnedTargets.DeviceCameraToggle,
+ ToggleState: "off");
+ var apps = new ElementInfo("button", "Let apps access your camera", [0.1, 0.3, 0.2, 0.1],
+ TargetId: "apps-off", AutomationId: CameraRecoveryPinnedTargets.AppCameraToggle,
+ IsEnabled: false, Targetable: false, ToggleState: "off");
+ var teams = new ElementInfo("button", "Microsoft Teams", [0.1, 0.5, 0.2, 0.1],
+ TargetId: "teams-off", AutomationId: CameraRecoveryPinnedTargets.PackagedTeamsCameraToggle,
+ IsEnabled: false, Targetable: false, ToggleState: "off");
+ var deviceObservation = LiveCameraRecoverySensing.AssessSettingsPermissions([device, apps, teams], false);
+ Check(deviceObservation.Target?.Kind == CameraRecoveryTargetKind.DeviceCameraPermission,
+ "disabled child toggles do not masquerade as policy when device access is off");
+ Check(LiveCameraRecoverySensing.AssessSettingsPermissions([device, apps, teams], true)
+ .Finding == CameraSettingsFinding.ManagedOrDisabled,
+ "policy disables permission approvals");
+ Check(LiveCameraRecoverySensing.AssessSettingsPermissions(
+ [device with { IsEnabled = false }, apps, teams], false).Target is null,
+ "disabled global toggle is never offered for execution");
+
+ device = device with { ToggleState = "on" };
+ apps = apps with { IsEnabled = true, Targetable = true };
+ var appsObservation = LiveCameraRecoverySensing.AssessSettingsPermissions([device, apps, teams], false);
+ Check(appsObservation.Target?.Kind == CameraRecoveryTargetKind.AppCameraPermission,
+ "app access is a distinct next scope");
+ apps = apps with { ToggleState = "on" };
+ teams = teams with { IsEnabled = true, Targetable = true };
+ var teamsObservation = LiveCameraRecoverySensing.AssessSettingsPermissions([device, apps, teams], false);
+ Check(teamsObservation.Target?.Kind == CameraRecoveryTargetKind.PackagedTeamsPermission,
+ "Teams permission is a distinct next scope");
+ teams = teams with { ToggleState = "on" };
+ var allowedObservation = LiveCameraRecoverySensing.AssessSettingsPermissions([device, apps, teams], false);
+ Check(allowedObservation.Finding == CameraSettingsFinding.PermissionOn, "all visible permissions are on");
+
+ foreach (var mode in new[] { CameraRecoveryInteractionMode.Guide, CameraRecoveryInteractionMode.Control })
+ {
+ var session = new CameraRecoverySession(mode);
+ session.Start();
+ session.ChooseTeamsWindow("teams-global", "Synthetic Teams");
+ session.ApplyTeamsObservation(new("teams-global", TeamsCameraFinding.PermissionMayBeOff,
+ deviceBlock.Detail, PermissionScope: deviceBlock.Scope));
+ Check(session.CanOpenSettings && !session.CanControlTarget && session.Detail == deviceBlock.Detail,
+ "preflight explains the blocker before exposing an action");
+ session.PrepareToOpenSettings();
+ session.MarkSettingsOpened();
+ foreach (var observation in new[] { deviceObservation, appsObservation, teamsObservation })
+ {
+ session.ApplySettingsObservation(observation);
+ Check(session.Target?.Kind == observation.Target?.Kind
+ && session.CanControlTarget == (mode == CameraRecoveryInteractionMode.Control)
+ && !session.CanVerifyTeams && !session.CanRestartTeams,
+ "each permission scope requires its own action and cannot claim readiness");
+ }
+ session.ApplySettingsObservation(allowedObservation);
+ Check(session.CanReturnToTeams && !session.LocalVerifierPassed, "permissions are not camera readiness");
+ session.MarkReturnedToTeams();
+ Check(session.CanInspectTeams && !session.CanControlTarget && !session.CanVerifyTeams,
+ "global permission recovery requires fresh Teams inspection");
+ session.ApplyTeamsObservation(new("teams-global", TeamsCameraFinding.CameraOff, "",
+ new("camera-fresh", "Turn camera on (Ctrl+Shift+O)", "camera", CameraRecoveryTargetKind.TeamsCameraButton)));
+ Check(session.CanControlTarget == (mode == CameraRecoveryInteractionMode.Control)
+ && session.Target?.Kind == CameraRecoveryTargetKind.TeamsCameraButton,
+ "camera-on is a separate action after permission recovery");
+ }
+
+ var wrongScope = SettingsSession();
+ wrongScope.ApplySettingsObservation(deviceObservation with
+ {
+ Target = deviceObservation.Target! with { Kind = CameraRecoveryTargetKind.PackagedTeamsPermission }
+ });
+ Check(wrongScope.State == CameraRecoveryState.Unsupported, "permission ID cannot authorize another scope");
+
+ var restoredBeforeInspection = new CameraRecoverySession(CameraRecoveryInteractionMode.Control);
+ restoredBeforeInspection.Start();
+ restoredBeforeInspection.ChooseTeamsWindow("teams-devices", "Synthetic Teams Devices");
+ restoredBeforeInspection.ApplyTeamsObservation(new("teams-devices", TeamsCameraFinding.PermissionMayBeOff,
+ deviceBlock.Detail, PermissionScope: deviceBlock.Scope));
+ restoredBeforeInspection.PrepareToOpenSettings();
+ restoredBeforeInspection.MarkSettingsOpened();
+ restoredBeforeInspection.ApplySettingsObservation(allowedObservation);
+ Check(restoredBeforeInspection.CanReturnToTeams && !restoredBeforeInspection.CanControlTarget,
+ "manually restored global permission has a read-only continuation");
+ restoredBeforeInspection.MarkReturnedToTeams();
+ restoredBeforeInspection.ApplyTeamsObservation(new("teams-devices",
+ TeamsCameraFinding.PermissionAlreadyOnOrDifferentCause, "All permissions are on."));
+ Check(restoredBeforeInspection.CanVerifyTeams && !restoredBeforeInspection.IsTerminal
+ && !restoredBeforeInspection.LocalVerifierPassed && !restoredBeforeInspection.CanControlTarget,
+ "post-restoration Devices surface continues to verification rather than wrong-cause terminal");
+ }
+
+ private static CameraRecoverySession StartedSession()
+ {
+ var session = new CameraRecoverySession();
+ session.Start();
+ session.ChooseTeamsWindow("teams-1", "Microsoft Teams");
+ return session;
+ }
+
+ private static CameraRecoverySession PermissionOnSession()
+ {
+ var session = SettingsSession();
+ session.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOff, "",
+ new CameraRecoveryTarget("settings-observation-2", "Microsoft Teams Currently in use",
+ CameraRecoveryPinnedTargets.PackagedTeamsCameraToggle,
+ CameraRecoveryTargetKind.PackagedTeamsPermission),
+ CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ session.ApplySettingsObservation(new(CameraSettingsFinding.PermissionOn, "",
+ Source: CameraSettingsObservationSource.ControlsOnly, ProbeValidated: true,
+ Page: CameraSettingsPage.CameraPrivacy));
+ session.MarkReturnedToTeams();
+ return session;
+ }
+
+ private static CameraRecoverySession SettingsSession()
+ {
+ var session = StartedSession();
+ session.ApplyTeamsObservation(new("teams-1", TeamsCameraFinding.PermissionMayBeOff, ""));
+ session.PrepareToOpenSettings();
+ session.MarkSettingsOpened();
+ return session;
+ }
+}
+
+public partial class App
+{
+ static App()
+ {
+ if (Environment.GetEnvironmentVariable("MSGUIDE_CAMERA_RECOVERY_TESTS") == "1")
+ CameraRecoveryTests.Run();
+ }
+}
diff --git a/desktop/CameraTargetRevalidation.cs b/desktop/CameraTargetRevalidation.cs
new file mode 100644
index 0000000..1901975
--- /dev/null
+++ b/desktop/CameraTargetRevalidation.cs
@@ -0,0 +1,64 @@
+namespace MSGuide.Desktop;
+
+internal enum CameraTargetFinding { Ready, AlreadyEnabled, WindowChanged, Expired, IdentityChanged, Unavailable }
+
+internal sealed record CameraTargetMatch(CameraTargetFinding Finding, ElementInfo? Element, string Detail);
+
+internal static class CameraTargetRevalidation
+{
+ internal static CameraTargetMatch Match(
+ WindowChoice approvedWindow, Native.RECT approvedBounds, ElementInfo approved,
+ CameraRecoveryTargetKind scope, DateTimeOffset observedAt,
+ WindowChoice currentWindow, Native.RECT currentBounds, IReadOnlyList elements,
+ DateTimeOffset now)
+ {
+ if (approvedWindow.Id != currentWindow.Id || approvedWindow.ClassName != currentWindow.ClassName
+ || approvedWindow.Title != currentWindow.Title || !approvedBounds.Same(currentBounds))
+ return new(CameraTargetFinding.WindowChanged, null,
+ "The camera window's identity, title, or bounds no longer match the observation. No action was started.");
+ if (!Safety.Fresh(observedAt, now))
+ return new(CameraTargetFinding.Expired, null,
+ "The camera observation expired before approval. No action was started; check the camera again.");
+ if (string.IsNullOrWhiteSpace(approved.ControlId))
+ return new(CameraTargetFinding.IdentityChanged, null,
+ "The camera control has no stable accessibility identity. No action was started.");
+ var matches = elements.Where(element => element.ControlId == approved.ControlId).ToArray();
+ if (matches.Length != 1)
+ return new(CameraTargetFinding.IdentityChanged, null,
+ "Teams or Windows replaced the camera control, or its identity is ambiguous. No action was started.");
+ var current = matches[0];
+ if (current.Role != approved.Role || current.AutomationId != approved.AutomationId
+ || current.FrameworkId != approved.FrameworkId || string.IsNullOrWhiteSpace(current.TargetId)
+ || !Safety.ValidBox(current.Box) || current.IsPassword || current.IsOffscreen
+ || !MatchesScope(current, scope))
+ return new(CameraTargetFinding.IdentityChanged, null,
+ "The accessible control no longer matches the approved camera scope. No action was started.");
+ if (IsOn(current, scope))
+ return new(CameraTargetFinding.AlreadyEnabled, current,
+ "The approved camera control is already on. No toggle was sent; checking the current camera state.");
+ if (!current.IsEnabled || !current.Targetable || !IsOff(current, scope))
+ return new(CameraTargetFinding.Unavailable, null,
+ "The approved camera control is not currently enabled, targetable, and off. No action was started.");
+ return new(CameraTargetFinding.Ready, current,
+ "The same approved camera control was freshly revalidated.");
+ }
+
+ internal static bool MatchesScope(ElementInfo element, CameraRecoveryTargetKind scope) =>
+ scope == CameraRecoveryTargetKind.TeamsCameraButton
+ ? CameraRecoveryPinnedTargets.IsTeamsCameraElement(element)
+ : CameraRecoveryPinnedTargets.IsPermission(scope)
+ && CameraRecoveryPinnedTargets.IsPermissionTarget(element.AutomationId, scope)
+ && element.Role is "button" or "checkbox";
+
+ internal static bool IsOff(ElementInfo element, CameraRecoveryTargetKind scope) =>
+ MatchesScope(element, scope) && (CameraRecoveryPinnedTargets.IsPermission(scope)
+ ? element.ToggleState == "off"
+ : CameraRecoveryPinnedTargets.IsTeamsTurnCameraOn(element.Label)
+ || CameraRecoveryPinnedTargets.IsTeamsCameraToggle(element.Label) && element.ToggleState == "off");
+
+ internal static bool IsOn(ElementInfo element, CameraRecoveryTargetKind scope) =>
+ MatchesScope(element, scope) && (CameraRecoveryPinnedTargets.IsPermission(scope)
+ ? element.ToggleState == "on"
+ : CameraRecoveryPinnedTargets.IsTeamsTurnCameraOff(element.Label)
+ || CameraRecoveryPinnedTargets.IsTeamsCameraToggle(element.Label) && element.ToggleState == "on");
+}
diff --git a/desktop/CameraTargetRevalidationTests.cs b/desktop/CameraTargetRevalidationTests.cs
new file mode 100644
index 0000000..3cefebe
--- /dev/null
+++ b/desktop/CameraTargetRevalidationTests.cs
@@ -0,0 +1,147 @@
+using System.Windows;
+using System.Windows.Automation;
+using System.Windows.Controls;
+using System.Windows.Interop;
+using System.Windows.Threading;
+
+namespace MSGuide.Desktop;
+
+internal static class CameraTargetRevalidationTests
+{
+ internal static async Task RunAsync()
+ {
+ var now = DateTimeOffset.UtcNow;
+ var window = new WindowChoice((nint)1234, 42, "Synthetic Camera settings", "SyntheticClass");
+ var bounds = new Native.RECT { Left = -900, Top = 20, Right = 100, Bottom = 720 };
+ var original = new ElementInfo("checkbox", "Camera access", [0.1, 0.2, 0.2, 0.1],
+ TargetId: "old-fingerprint", AutomationId: CameraRecoveryPinnedTargets.DeviceCameraToggle,
+ FrameworkId: "WPF", ToggleState: "off", ControlId: "same-control");
+ CameraTargetMatch Match(params ElementInfo[] current) => CameraTargetRevalidation.Match(
+ window, bounds, original, CameraRecoveryTargetKind.DeviceCameraPermission, now,
+ window, bounds, current, now);
+
+ var redrawn = original with
+ {
+ Label = "Camera access - available to permitted apps",
+ Box = [0.12, 0.24, 0.2, 0.1], TargetId = "fresh-fingerprint"
+ };
+ IntegrationTests.Require(Match(original).Finding == CameraTargetFinding.Ready
+ && Match(redrawn) is { Finding: CameraTargetFinding.Ready, Element.TargetId: "fresh-fingerprint" }
+ && redrawn.ControlId == original.ControlId && redrawn.TargetId != original.TargetId);
+ IntegrationTests.Require(Match(redrawn with { ToggleState = "on" }).Finding == CameraTargetFinding.AlreadyEnabled);
+ foreach (var invalid in new[]
+ {
+ redrawn with { ControlId = "replacement-control" },
+ redrawn with { ControlId = null }, redrawn with { TargetId = null },
+ redrawn with { AutomationId = CameraRecoveryPinnedTargets.AppCameraToggle },
+ redrawn with { Role = "edit" }, redrawn with { FrameworkId = "other-provider" },
+ redrawn with { IsEnabled = false }, redrawn with { Targetable = false },
+ redrawn with { IsPassword = true }, redrawn with { IsOffscreen = true },
+ redrawn with { ToggleState = "unknown" }, redrawn with { Box = [double.NaN, 0, 1, 1] }
+ })
+ IntegrationTests.Require(Match(invalid).Finding is not (CameraTargetFinding.Ready or CameraTargetFinding.AlreadyEnabled));
+ IntegrationTests.Require(Match().Finding == CameraTargetFinding.IdentityChanged
+ && Match(redrawn, redrawn).Finding == CameraTargetFinding.IdentityChanged);
+ foreach (var changedWindow in new[]
+ {
+ window with { Handle = (nint)2222 }, window with { ProcessId = 43 },
+ window with { ClassName = "replaced" }, window with { Title = "Another resource" }
+ })
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, bounds, original,
+ CameraRecoveryTargetKind.DeviceCameraPermission, now, changedWindow, bounds, [redrawn], now)
+ .Finding == CameraTargetFinding.WindowChanged);
+ var movedBounds = bounds;
+ movedBounds.Left++;
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, bounds, original,
+ CameraRecoveryTargetKind.DeviceCameraPermission, now, window, movedBounds, [redrawn], now)
+ .Finding == CameraTargetFinding.WindowChanged);
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, bounds, original,
+ CameraRecoveryTargetKind.DeviceCameraPermission, now.AddSeconds(-60), window, bounds, [redrawn], now)
+ .Finding == CameraTargetFinding.Expired);
+ var camera = original with
+ {
+ AutomationId = "camera-toggle", Label = "Turn camera on (Ctrl+Shift+O)", Role = "button"
+ };
+ var enabled = camera with { Label = "Turn camera off (Ctrl+Shift+O)", TargetId = "camera-now-on" };
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, bounds, camera,
+ CameraRecoveryTargetKind.TeamsCameraButton, now, window, bounds, [enabled], now)
+ .Finding == CameraTargetFinding.AlreadyEnabled);
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, bounds, camera,
+ CameraRecoveryTargetKind.TeamsCameraButton, now, window, bounds,
+ [camera with { Label = "Turn microphone on" }], now).Finding == CameraTargetFinding.IdentityChanged);
+ var satisfied = await DesktopAction.RunBounded((_, _) =>
+ new(false, true, "Synthetic already-enabled observation.", StateAlreadySatisfied: true), CancellationToken.None);
+ IntegrationTests.Require(!satisfied.Invoked && satisfied.OutcomeKnown && satisfied.StateAlreadySatisfied);
+ }
+
+ internal static async Task RunNativeAsync(CancellationToken ct)
+ {
+ var toggle = new CheckBox
+ {
+ Content = "Camera access", IsChecked = false,
+ HorizontalAlignment = HorizontalAlignment.Left, VerticalAlignment = VerticalAlignment.Top
+ };
+ AutomationProperties.SetAutomationId(toggle, CameraRecoveryPinnedTargets.DeviceCameraToggle);
+ var panel = new Grid();
+ panel.Children.Add(toggle);
+ var fixture = new Window
+ {
+ Title = "MSGuide owned revalidation fixture", Width = 520, Height = 360,
+ Content = panel, ShowActivated = false
+ };
+ int toggles = 0;
+ toggle.Checked += (_, _) => toggles++;
+ toggle.Unchecked += (_, _) => toggles++;
+ try
+ {
+ fixture.Show();
+ await Dispatcher.Yield(DispatcherPriority.ApplicationIdle);
+ var window = new WindowChoice(new WindowInteropHelper(fixture).Handle,
+ (uint)Environment.ProcessId, fixture.Title);
+ using var observed = await CaptureService.InspectCameraControls(window, ct);
+ var original = observed.Elements.Single(element =>
+ element.AutomationId == CameraRecoveryPinnedTargets.DeviceCameraToggle);
+ toggle.Margin = new Thickness(45, 25, 0, 0);
+ toggle.Content = "Camera access - synthetic description updated";
+ await Dispatcher.Yield(DispatcherPriority.ApplicationIdle);
+ using var refreshed = await CaptureService.InspectCameraControls(window, ct);
+ var fresh = refreshed.Elements.Single(element => element.AutomationId == original.AutomationId);
+ IntegrationTests.Require(observed.Rect.Same(refreshed.Rect) && original.ControlId == fresh.ControlId
+ && original.TargetId != fresh.TargetId
+ && !refreshed.Elements.Any(element => element.TargetId == original.TargetId));
+ var match = CameraTargetRevalidation.Match(window, observed.Rect, original,
+ CameraRecoveryTargetKind.DeviceCameraPermission, observed.CapturedAt, window, refreshed.Rect,
+ refreshed.Elements, DateTimeOffset.UtcNow);
+ IntegrationTests.Require(match.Finding == CameraTargetFinding.Ready && match.Element == fresh);
+ var result = await DesktopAction.RunBounded((token, beginInvocation) =>
+ {
+ var raw = AutomationEvidence.FindUniqueTarget(window, refreshed.Rect, fresh.TargetId!,
+ fresh.Label, fresh.AutomationId, token);
+ IntegrationTests.Require(raw is not null && window.ProcessId == Environment.ProcessId
+ && raw.Current.ProcessId == Environment.ProcessId);
+ var pattern = raw!.GetCurrentPattern(TogglePattern.Pattern) as TogglePattern;
+ IntegrationTests.Require(pattern is not null && pattern.Current.ToggleState == ToggleState.Off);
+ if (!beginInvocation()) return new(false, true, "Synthetic toggle cancelled.");
+ pattern!.Toggle();
+ return new(true, true, "Owned synthetic control toggled once.");
+ }, ct);
+ await Dispatcher.Yield(DispatcherPriority.ApplicationIdle);
+ IntegrationTests.Require(result.Invoked && result.OutcomeKnown && toggles == 1 && toggle.IsChecked == true);
+ using var enabled = await CaptureService.InspectCameraControls(window, ct);
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, observed.Rect, original,
+ CameraRecoveryTargetKind.DeviceCameraPermission, observed.CapturedAt, window, enabled.Rect,
+ enabled.Elements, DateTimeOffset.UtcNow).Finding == CameraTargetFinding.AlreadyEnabled);
+ panel.Children.Clear();
+ var replacement = new CheckBox { Content = "Camera access", IsChecked = false };
+ AutomationProperties.SetAutomationId(replacement, CameraRecoveryPinnedTargets.DeviceCameraToggle);
+ panel.Children.Add(replacement);
+ await Dispatcher.Yield(DispatcherPriority.ApplicationIdle);
+ using var replaced = await CaptureService.InspectCameraControls(window, ct);
+ IntegrationTests.Require(CameraTargetRevalidation.Match(window, observed.Rect, original,
+ CameraRecoveryTargetKind.DeviceCameraPermission, observed.CapturedAt, window, replaced.Rect,
+ replaced.Elements, DateTimeOffset.UtcNow).Finding == CameraTargetFinding.IdentityChanged
+ && toggles == 1 && replacement.IsChecked == false);
+ }
+ finally { fixture.Close(); }
+ }
+}
diff --git a/desktop/CameraWindowDiscovery.cs b/desktop/CameraWindowDiscovery.cs
new file mode 100644
index 0000000..ac48276
--- /dev/null
+++ b/desktop/CameraWindowDiscovery.cs
@@ -0,0 +1,63 @@
+namespace MSGuide.Desktop;
+
+internal enum CameraSurfaceFinding { MeetingCamera, NoCamera, Incomplete }
+
+internal sealed record CameraWindowSelection(WindowChoice? Window, string Detail);
+
+internal interface ICameraWindowDiscovery
+{
+ Task InspectCameraSurfaceAsync(WindowChoice window, CancellationToken cancellationToken);
+}
+
+internal static class CameraWindowDiscovery
+{
+ private const int MaxCandidates = 6;
+
+ internal static async Task SelectAsync(
+ IReadOnlyList candidates, WindowChoice? invoked,
+ ICameraWindowDiscovery discovery, CancellationToken ct)
+ {
+ ct.ThrowIfCancellationRequested();
+ var windows = candidates.Where(window => window.IsMicrosoftTeamsWindow)
+ .DistinctBy(window => window.Id).ToArray();
+ if (windows.Length == 0)
+ return new(null, "Open a Teams meeting or prejoin with its camera controls visible, then check again.");
+ if (windows.Length > MaxCandidates)
+ return new(null, "Too many Teams windows are open to inspect safely. Close unused windows or choose the meeting explicitly.");
+
+ var matches = new List();
+ bool incomplete = false;
+ foreach (var window in windows.OrderByDescending(window => window.Id == invoked?.Id))
+ {
+ ct.ThrowIfCancellationRequested();
+ var result = await discovery.InspectCameraSurfaceAsync(window, ct);
+ ct.ThrowIfCancellationRequested();
+ if (result == CameraSurfaceFinding.MeetingCamera)
+ {
+ if (window.Id == invoked?.Id)
+ return new(window, "Using the Teams meeting or prejoin you invoked MSGuide from.");
+ matches.Add(window);
+ }
+ else if (result == CameraSurfaceFinding.Incomplete) incomplete = true;
+ }
+ if (incomplete)
+ return new(null, "A Teams window could not be inspected completely. Choose the intended meeting; no camera action was prepared.");
+ return matches.Count switch
+ {
+ 1 => new(matches[0], "Found the Teams meeting or prejoin with a visible camera control."),
+ > 1 => new(null, "More than one Teams meeting has camera controls. Choose which meeting to fix."),
+ _ => new(null, "No visible meeting or prejoin camera control was found. Open that Teams surface, then check again.")
+ };
+ }
+
+ internal static CameraSurfaceFinding Assess(IEnumerable elements)
+ {
+ int count = elements.Count(CameraRecoveryPinnedTargets.IsTeamsCameraElement);
+ return count switch
+ {
+ 0 => CameraSurfaceFinding.NoCamera,
+ 1 => CameraSurfaceFinding.MeetingCamera,
+ _ => CameraSurfaceFinding.Incomplete
+ };
+ }
+}
diff --git a/desktop/CameraWindowDiscoveryTests.cs b/desktop/CameraWindowDiscoveryTests.cs
new file mode 100644
index 0000000..e169b2d
--- /dev/null
+++ b/desktop/CameraWindowDiscoveryTests.cs
@@ -0,0 +1,106 @@
+using System.Windows;
+using System.Windows.Automation;
+using System.Windows.Controls;
+using System.Windows.Interop;
+using System.Windows.Threading;
+
+namespace MSGuide.Desktop;
+
+internal static class CameraWindowDiscoveryTests
+{
+ private sealed class Discovery(Func inspect) : ICameraWindowDiscovery
+ {
+ internal int Reads;
+ public Task InspectCameraSurfaceAsync(WindowChoice window, CancellationToken ct)
+ {
+ ct.ThrowIfCancellationRequested();
+ Reads++;
+ return Task.FromResult(inspect(window));
+ }
+ }
+
+ internal static async Task RunAsync()
+ {
+ var home = new WindowChoice((nint)1, 1, "Chat | Microsoft Teams", "Synthetic");
+ var meeting = new WindowChoice((nint)2, 1, "Meeting | Microsoft Teams", "Synthetic");
+ var other = new WindowChoice((nint)3, 1, "Other meeting | Microsoft Teams", "Synthetic");
+ var discovery = new Discovery(window => window == meeting
+ ? CameraSurfaceFinding.MeetingCamera : CameraSurfaceFinding.NoCamera);
+ foreach (var invoked in new WindowChoice?[] { null, home, meeting })
+ {
+ var found = await CameraWindowDiscovery.SelectAsync([home, meeting], invoked, discovery, CancellationToken.None);
+ IntegrationTests.Require(found.Window == meeting);
+ }
+ discovery = new(window => window == home ? CameraSurfaceFinding.NoCamera : CameraSurfaceFinding.MeetingCamera);
+ IntegrationTests.Require((await CameraWindowDiscovery.SelectAsync(
+ [home, meeting, other], home, discovery, CancellationToken.None)).Window is null);
+ IntegrationTests.Require((await CameraWindowDiscovery.SelectAsync(
+ [home, meeting, other], meeting, discovery, CancellationToken.None)).Window == meeting);
+ discovery = new(window => window == meeting ? CameraSurfaceFinding.MeetingCamera : CameraSurfaceFinding.Incomplete);
+ IntegrationTests.Require((await CameraWindowDiscovery.SelectAsync(
+ [home, meeting], null, discovery, CancellationToken.None)).Window is null);
+ discovery = new(_ => CameraSurfaceFinding.NoCamera);
+ IntegrationTests.Require((await CameraWindowDiscovery.SelectAsync(
+ [home], home, discovery, CancellationToken.None)).Window is null);
+ discovery = new(_ => CameraSurfaceFinding.MeetingCamera);
+ var excessive = Enumerable.Range(1, 7).Select(index => home with { Handle = (nint)index }).ToArray();
+ IntegrationTests.Require((await CameraWindowDiscovery.SelectAsync(
+ excessive, null, discovery, CancellationToken.None)).Window is null && discovery.Reads == 0);
+ using var cancelled = new CancellationTokenSource();
+ cancelled.Cancel();
+ bool rejected = false;
+ try { await CameraWindowDiscovery.SelectAsync([meeting], null, discovery, cancelled.Token); }
+ catch (OperationCanceledException) { rejected = true; }
+ IntegrationTests.Require(rejected && discovery.Reads == 0);
+
+ var camera = new ElementInfo("button", "Turn camera on (Ctrl+Shift+O)", [0.1, 0.2, 0.3, 0.1]);
+ IntegrationTests.Require(CameraWindowDiscovery.Assess([camera]) == CameraSurfaceFinding.MeetingCamera);
+ IntegrationTests.Require(CameraWindowDiscovery.Assess(
+ [camera with { IsEnabled = false, Targetable = false }]) == CameraSurfaceFinding.MeetingCamera);
+ foreach (var invalid in new[]
+ {
+ camera with { Role = "text" }, camera with { IsOffscreen = true }, camera with { IsPassword = true },
+ camera with { Label = "Turn camera on (for everyone)" }
+ })
+ IntegrationTests.Require(CameraWindowDiscovery.Assess([invalid]) == CameraSurfaceFinding.NoCamera);
+ IntegrationTests.Require(CameraWindowDiscovery.Assess([camera, camera]) == CameraSurfaceFinding.Incomplete);
+ }
+
+ internal static async Task RunNativeAsync(CancellationToken ct)
+ {
+ var button = new Button { Content = "Turn camera on (Ctrl+Shift+O)", IsEnabled = false };
+ AutomationProperties.SetName(button, "Turn camera on (Ctrl+Shift+O)");
+ var meeting = new Window
+ {
+ Title = "Owned camera discovery meeting | Microsoft Teams",
+ Width = 480, Height = 320, ShowActivated = false, Content = button
+ };
+ var home = new Window
+ {
+ Title = "Owned camera discovery chat | Microsoft Teams",
+ Width = 480, Height = 320, ShowActivated = false,
+ Content = new TextBlock { Text = "Turn camera on (Ctrl+Shift+O)" }
+ };
+ var overlay = new OverlayWindow();
+ int clicks = 0;
+ button.Click += (_, _) => clicks++;
+ try
+ {
+ home.Show();
+ meeting.Show();
+ await Dispatcher.Yield(DispatcherPriority.ApplicationIdle);
+ var meetingChoice = new WindowChoice(new WindowInteropHelper(meeting).Handle,
+ (uint)Environment.ProcessId, meeting.Title);
+ var homeChoice = new WindowChoice(new WindowInteropHelper(home).Handle,
+ (uint)Environment.ProcessId, home.Title);
+ using (var controls = await CaptureService.InspectCameraControls(meetingChoice, ct))
+ IntegrationTests.Require(controls.Valid() && controls.Png.Length == 0 && controls.Preview is null
+ && CameraWindowDiscovery.Assess(controls.Elements) == CameraSurfaceFinding.MeetingCamera);
+ var sensing = new LiveCameraRecoverySensing(overlay);
+ var found = await CameraWindowDiscovery.SelectAsync(
+ [homeChoice, meetingChoice], homeChoice, sensing, ct);
+ IntegrationTests.Require(found.Window == meetingChoice && clicks == 0 && !overlay.IsVisible);
+ }
+ finally { meeting.Close(); home.Close(); overlay.Close(); }
+ }
+}
diff --git a/desktop/CaptureProbe/CaptureProbeProgram.cs b/desktop/CaptureProbe/CaptureProbeProgram.cs
new file mode 100644
index 0000000..7a8eebe
--- /dev/null
+++ b/desktop/CaptureProbe/CaptureProbeProgram.cs
@@ -0,0 +1,135 @@
+using System.Globalization;
+using System.IO;
+using System.Text.Json;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Interop;
+using System.Windows.Media;
+using MSGuide.Desktop;
+
+internal static class CaptureProbeProgram
+{
+ [STAThread]
+ private static int Main(string[] args)
+ {
+ string? hwndValue = null, output = null;
+ bool selfTest = false;
+ for (int i = 0; i < args.Length; i++)
+ {
+ if (args[i] == "--hwnd" && hwndValue is null && ++i < args.Length) hwndValue = args[i];
+ else if (args[i] == "--output" && output is null && ++i < args.Length) output = args[i];
+ else if (args[i] == "--self-test" && !selfTest) selfTest = true;
+ else return Fail(output, "invalid-arguments");
+ }
+ if (output is null || selfTest == (hwndValue is not null))
+ return Fail(output, "invalid-arguments");
+ if (selfTest) return RunSelfTest(output);
+ if (!TryHandle(hwndValue!, out var hwnd)) return Fail(output, "invalid-arguments");
+ try
+ {
+ CaptureProbe.WriteJson(hwnd, output);
+ return 0;
+ }
+ catch (OperationCanceledException) { return Fail(output, "cancelled"); }
+ catch (InvalidOperationException) { return Fail(output, "window-unavailable-or-changed"); }
+ catch (Exception) { return Fail(output, "probe-error"); }
+ }
+
+ private static int RunSelfTest(string output)
+ {
+ int exitCode = 1;
+ var app = new Application { ShutdownMode = ShutdownMode.OnExplicitShutdown };
+ var window = new Window
+ {
+ Width = 520,
+ Height = 360,
+ Title = "MSGuide WGC Probe",
+ ShowActivated = false,
+ Content = new Border
+ {
+ Background = Brushes.White,
+ Padding = new Thickness(32),
+ Child = new StackPanel
+ {
+ Children =
+ {
+ new TextBlock
+ {
+ Text = "Windows Graphics Capture",
+ FontSize = 28,
+ Foreground = Brushes.Black
+ },
+ new Button
+ {
+ Content = "Synthetic camera control",
+ Margin = new Thickness(0, 30, 0, 0)
+ }
+ }
+ }
+ }
+ };
+ window.ContentRendered += async (_, _) =>
+ {
+ try
+ {
+ var hwnd = new WindowInteropHelper(window).Handle;
+ Native.GetWindowThreadProcessId(hwnd, out var pid);
+ var choice = new WindowChoice(hwnd, pid, window.Title);
+ using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+ using var snapshot = await CaptureService.Capture(choice, deadline.Token);
+ if (!snapshot.Valid() || snapshot.Png.Length <= 8
+ || !snapshot.Png.Take(8).SequenceEqual(new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 })
+ || snapshot.Elements.Any(element => string.IsNullOrWhiteSpace(element.TargetId)))
+ throw new InvalidOperationException();
+ var report = await Task.Run(() => CaptureProbe.RunWgcOnly(hwnd));
+ Write(output, report);
+ exitCode = report.Attempts is [{ Backend: "WindowsGraphicsCapture", Accepted: true }]
+ && report.Automation.RootMatched ? 0 : 1;
+ }
+ catch (Exception) { Fail(output, "probe-error"); }
+ finally
+ {
+ window.Close();
+ app.Shutdown();
+ }
+ };
+ window.Show();
+ app.Run();
+ return exitCode;
+ }
+
+ private static bool TryHandle(string value, out nint hwnd)
+ {
+ bool hex = value.StartsWith("0x", StringComparison.OrdinalIgnoreCase);
+ string digits = hex ? value[2..] : value;
+ bool parsed = long.TryParse(digits, hex ? NumberStyles.HexNumber : NumberStyles.Integer,
+ CultureInfo.InvariantCulture, out long handle);
+ hwnd = new nint(handle);
+ return parsed && handle != 0;
+ }
+
+ private static int Fail(string? output, string code)
+ {
+ if (output is not null)
+ {
+ try
+ {
+ File.WriteAllText(output, JsonSerializer.Serialize(new
+ {
+ version = 1,
+ capturedAt = DateTimeOffset.UtcNow,
+ completed = false,
+ failure = code
+ }, new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true })
+ + Environment.NewLine);
+ }
+ catch { }
+ }
+ return 1;
+ }
+
+ private static void Write(string output, CaptureProbeReport report) =>
+ File.WriteAllText(output, JsonSerializer.Serialize(report,
+ new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true })
+ + Environment.NewLine);
+}
diff --git a/desktop/CaptureProbe/MSGuide.CaptureProbe.csproj b/desktop/CaptureProbe/MSGuide.CaptureProbe.csproj
new file mode 100644
index 0000000..48a3bc8
--- /dev/null
+++ b/desktop/CaptureProbe/MSGuide.CaptureProbe.csproj
@@ -0,0 +1,15 @@
+
+
+ Exe
+ net10.0-windows10.0.19041.0
+ true
+ enable
+ enable
+ false
+ false
+ CaptureProbeProgram
+
+
+
+
+
diff --git a/desktop/CaptureProbe/packages.lock.json b/desktop/CaptureProbe/packages.lock.json
new file mode 100644
index 0000000..b395da2
--- /dev/null
+++ b/desktop/CaptureProbe/packages.lock.json
@@ -0,0 +1,60 @@
+{
+ "version": 1,
+ "dependencies": {
+ "net10.0-windows10.0.19041": {
+ "Microsoft.Extensions.AI.Abstractions": {
+ "type": "Transitive",
+ "resolved": "10.2.0",
+ "contentHash": "ONGlIBht0ygEdKc0bCt9XWUiq19/460dAu7fCKzPM34OFJSMQIohEDTJwjCJ8vVp8znNakloc9xFF9+R/eDCYQ=="
+ },
+ "NAudio.Core": {
+ "type": "Transitive",
+ "resolved": "2.2.1",
+ "contentHash": "GgkdP6K/7FqXFo7uHvoqGZTJvW4z8g2IffhOO4JHaLzKCdDOUEzVKtveoZkCuUX8eV2HAINqi7VFqlFndrnz/g=="
+ },
+ "NAudio.WinMM": {
+ "type": "Transitive",
+ "resolved": "2.2.1",
+ "contentHash": "xFHRFwH4x6aq3IxRbewvO33ugJRvZFEOfO62i7uQJRUNW2cnu6BeBTHUS0JD5KBucZbHZaYqxQG8dwZ47ezQuQ==",
+ "dependencies": {
+ "NAudio.Core": "2.2.1"
+ }
+ },
+ "System.Speech": {
+ "type": "Transitive",
+ "resolved": "10.0.0",
+ "contentHash": "nHlCdjoeReiSx4kzgPEwPRlbOAyuVFwndXTZZT+CLnHb8UIfB+YMgpQjQ7faYvs3xU8wyvpb6dt+U+HwdlbG7w=="
+ },
+ "Whisper.net": {
+ "type": "Transitive",
+ "resolved": "1.9.1",
+ "contentHash": "82WAKyyMun44cAXE80sP7vIra1oYhbVp2WrnqJm0CISTfMAvint1k/umt0SvsQH+IkDyekZ3BgMdS3Rag9SZ5A==",
+ "dependencies": {
+ "Microsoft.Extensions.AI.Abstractions": "10.2.0"
+ }
+ },
+ "Whisper.net.Runtime": {
+ "type": "Transitive",
+ "resolved": "1.9.1",
+ "contentHash": "TipPJkry69VwJ3XL38U082QK5Trqte6NgSOoosuyj/C1eRHabmBbLQ3ujjcDryg5pkXT7wXwAKU5+PxGIJyxkQ==",
+ "dependencies": {
+ "Whisper.net.Runtime.Metal": "1.9.1"
+ }
+ },
+ "Whisper.net.Runtime.Metal": {
+ "type": "Transitive",
+ "resolved": "1.9.1",
+ "contentHash": "Ut5tZyCUwXMVdThoLvdvQ7GyLwjRMreR/p3LuMJvRiXidQ2AXxx2Tv8UOGtMxvL395AC3+IkHdN/kMfd+EPSEw=="
+ },
+ "msguide.desktop": {
+ "type": "Project",
+ "dependencies": {
+ "NAudio.WinMM": "[2.2.1, )",
+ "System.Speech": "[10.0.0, )",
+ "Whisper.net": "[1.9.1, )",
+ "Whisper.net.Runtime": "[1.9.1, )"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/desktop/CaptureService.cs b/desktop/CaptureService.cs
index e324a65..abf83f1 100644
--- a/desktop/CaptureService.cs
+++ b/desktop/CaptureService.cs
@@ -9,8 +9,10 @@
namespace MSGuide.Desktop;
public sealed class Snapshot(WindowChoice window, Native.RECT rect, DateTimeOffset captured,
- byte[] png, BitmapSource preview, ElementInfo[] elements, string text, string note) : IDisposable
+ byte[] png, BitmapSource? preview, ElementInfo[] elements, string text, string note,
+ bool automationComplete = true, string? resourceId = null, string? application = null) : IDisposable
{
+ private bool disposed;
public string Id { get; } = Guid.NewGuid().ToString();
public WindowChoice Window { get; } = window;
public Native.RECT Rect { get; } = rect;
@@ -20,30 +22,49 @@ public sealed class Snapshot(WindowChoice window, Native.RECT rect, DateTimeOffs
public ElementInfo[] Elements { get; private set; } = elements;
public string Text { get; private set; } = text;
public string Note { get; } = note;
- public bool Valid() => Preview is not null && Safety.Fresh(CapturedAt, DateTimeOffset.UtcNow) && Window.Matches()
- && Native.GetWindowRect(Window.Handle, out var now) && Rect.Same(now);
+ public bool AutomationComplete { get; } = automationComplete;
+ public string? ResourceId { get; } = resourceId;
+ private string ApplicationName { get; } = application ?? window.Title;
+ public bool Valid() => !disposed && Safety.Fresh(CapturedAt, DateTimeOffset.UtcNow) && Window.Matches()
+ && Native.GetWindowRect(Window.Handle, out var now) && Rect.Same(now)
+ && (ResourceId is null || (ResourceId.StartsWith("browser-", StringComparison.Ordinal)
+ ? Native.Title(Window.Handle) == ApplicationName
+ : ResourceId == AutomationEvidence.ResourceId(Window, Native.Title(Window.Handle), Elements)));
public Observation Observation(bool image)
{
- if (Preview is null) throw new ObjectDisposedException(nameof(Snapshot));
- return new(Id, Window.Id, Window.Title[..Math.Min(Window.Title.Length, 256)], CapturedAt,
- Preview.PixelWidth, Preview.PixelHeight, Text, Elements, image ? Convert.ToBase64String(Png) : null);
+ if (disposed) throw new ObjectDisposedException(nameof(Snapshot));
+ return new(Id, Window.Id, ApplicationName[..Math.Min(ApplicationName.Length, 256)], CapturedAt,
+ Preview?.PixelWidth ?? Rect.Width, Preview?.PixelHeight ?? Rect.Height, Text, Elements,
+ image && Png.Length > 0 ? Convert.ToBase64String(Png) : null, AutomationComplete, ResourceId);
}
public void Dispose()
{
+ disposed = true;
Array.Clear(Png); Png = []; Preview = null; Elements = []; Text = "";
}
}
public static class CaptureService
{
+ private static readonly ISelectedWindowFrameCapture FrameCapture = new WindowsGraphicsCaptureFrameCapture();
+
// ponytail: one outstanding native capture. A stuck provider cannot queue more workers;
- // process-isolate PrintWindow/UIA if support for unresponsive applications becomes required.
+ // process-isolate UIA if support for unresponsive applications becomes required.
private static int busy;
internal static Task WhenIdle { get; private set; } = Task.CompletedTask;
- public static async Task Capture(WindowChoice window, CancellationToken ct)
+ public static Task Capture(WindowChoice window, CancellationToken ct, bool includeImage = true) =>
+ CaptureBounded(window, ct, includeImage, cameraControls: false);
+
+ internal static Task InspectCameraControls(WindowChoice window, CancellationToken ct) =>
+ CaptureBounded(window, ct, includeImage: false, cameraControls: true);
+
+ private static async Task CaptureBounded(
+ WindowChoice window, CancellationToken ct, bool includeImage, bool cameraControls)
{
ct.ThrowIfCancellationRequested();
+ if (DesktopAction.IsBusy)
+ throw new InvalidOperationException("A native action is still returning. No new capture or action can start until it finishes.");
if (Interlocked.CompareExchange(ref busy, 1, 0) != 0)
throw new InvalidOperationException("A previous window capture is still returning. Use another application after it finishes, or restart MSGuide.");
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
@@ -54,7 +75,7 @@ public static async Task Capture(WindowChoice window, CancellationToke
Snapshot? result = null;
try
{
- result = CaptureCore(window, captureToken);
+ result = CaptureCore(window, captureToken, includeImage, cameraControls);
captureToken.ThrowIfCancellationRequested();
return result;
}
@@ -77,12 +98,12 @@ public static async Task Capture(WindowChoice window, CancellationToke
{
// A native call cannot be aborted safely. Drop and clear anything it returns later.
_ = task.ContinueWith(t => { if (t.IsCompletedSuccessfully) t.Result.Dispose(); else _ = t.Exception; }, TaskScheduler.Default);
- if (!ct.IsCancellationRequested) throw new InvalidOperationException("Window capture timed out. No snapshot was sent. This application may not support PrintWindow/UI Automation.");
+ if (!ct.IsCancellationRequested) throw new InvalidOperationException("Window capture timed out. No snapshot was sent. This application may not support Windows Graphics Capture/UI Automation.");
throw;
}
}
- private static Snapshot CaptureCore(WindowChoice window, CancellationToken ct)
+ private static Snapshot CaptureCore(WindowChoice window, CancellationToken ct, bool includeImage, bool cameraControls)
{
var previousDpi = Native.SetThreadDpiAwarenessContext(new nint(-4));
byte[]? png = null;
@@ -96,69 +117,49 @@ private static Snapshot CaptureCore(WindowChoice window, CancellationToken ct)
|| (long)rect.Width * rect.Height > 32_000_000)
throw new InvalidOperationException("Unsupported window dimensions. Resize the selected window and retry.");
var captured = DateTimeOffset.UtcNow;
- var preview = ReadWindow(window.Handle, rect, ct);
- ct.ThrowIfCancellationRequested();
- png = Encode(preview);
- while (png.Length > 2_000_000 && preview.PixelWidth > 320 && preview.PixelHeight > 200)
+ string resourceTitle = Native.Title(window.Handle);
+ bool browser = AutomationEvidence.IsSupportedBrowser(window);
+ string? browserResource = browser ? AutomationEvidence.ReadBrowserResourceId(window, ct) : null;
+ BitmapSource? preview = null;
+ png = [];
+ if (includeImage)
{
+ preview = ReadWindow(window, rect, ct);
ct.ThrowIfCancellationRequested();
- Array.Clear(png);
- preview = Resize(preview, 0.75);
png = Encode(preview);
+ while (png.Length > 2_000_000 && preview.PixelWidth > 320 && preview.PixelHeight > 200)
+ {
+ ct.ThrowIfCancellationRequested();
+ Array.Clear(png);
+ preview = Resize(preview, 0.75);
+ png = Encode(preview);
+ }
+ if (png.Length > 2_000_000) throw new InvalidOperationException("PNG exceeds the 2 MB limit; select a smaller window.");
}
- if (png.Length > 2_000_000) throw new InvalidOperationException("PNG exceeds the 2 MB limit; select a smaller window.");
- var (elements, text, note) = ReadAutomation(window, rect, ct);
+ var read = ReadAutomation(window, rect, ct,
+ maxDepth: cameraControls ? 32 : AutomationEvidence.ScanDepthLimit);
+ if (cameraControls) read.RequireComplete();
+ var (elements, text, note, complete) = read;
ct.ThrowIfCancellationRequested();
if (!window.Matches() || !Native.GetWindowRect(window.Handle, out var after) || !rect.Same(after))
throw new InvalidOperationException("Window changed during capture. Capture and review again.");
- return new(window, rect, captured, png, preview, elements, text, note);
+ if (resourceTitle != Native.Title(window.Handle))
+ throw new InvalidOperationException("The selected resource changed during capture. Review it before continuing.");
+ if (browser && browserResource != AutomationEvidence.ReadBrowserResourceId(window, ct))
+ throw new InvalidOperationException("The browser resource changed during capture. Review it before continuing.");
+ return new(window, rect, captured, png, preview, elements, text, note, complete,
+ browser ? browserResource : AutomationEvidence.ResourceId(window, resourceTitle, elements), resourceTitle);
}
catch { if (png is not null) Array.Clear(png); throw; }
finally { Native.SetThreadDpiAwarenessContext(previousDpi); }
}
- private static BitmapSource ReadWindow(nint hwnd, Native.RECT rect, CancellationToken ct)
+ private static BitmapSource ReadWindow(WindowChoice window, Native.RECT rect, CancellationToken ct)
{
- var dc = Native.CreateCompatibleDC(0);
- if (dc == 0) throw new InvalidOperationException("Cannot allocate a window capture context.");
- nint bitmap = 0, old = 0, bits = 0;
- byte[] pixels = new byte[checked(rect.Width * rect.Height * 4)];
- try
- {
- var info = new Native.BITMAPINFO { Size = 40, Width = rect.Width, Height = -rect.Height, Planes = 1, BitCount = 32 };
- bitmap = Native.CreateDIBSection(dc, ref info, 0, out bits, 0, 0);
- if (bitmap == 0 || bits == 0) throw new InvalidOperationException("Cannot allocate a window bitmap.");
- old = Native.SelectObject(dc, bitmap);
- Marshal.Copy(pixels, 0, bits, pixels.Length);
- // Only the approved HWND renders into this private bitmap. NEVER copy the desktop.
- ct.ThrowIfCancellationRequested();
- if (!Native.PrintWindow(hwnd, dc, 2))
- throw new InvalidOperationException("This window does not support PrintWindow capture. No desktop fallback is used.");
- ct.ThrowIfCancellationRequested();
- Marshal.Copy(bits, pixels, 0, pixels.Length);
- int min = 255, max = 0;
- for (int y = rect.Height / 10; y < rect.Height * 9 / 10; y += Math.Max(1, rect.Height / 80))
- for (int x = rect.Width / 10; x < rect.Width * 9 / 10; x += Math.Max(1, rect.Width / 80))
- {
- int i = (y * rect.Width + x) * 4;
- int light = (pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3;
- min = Math.Min(min, light); max = Math.Max(max, light);
- }
- if (max < 8 || max - min < 3)
- throw new InvalidOperationException("Capture appears blank, protected, or unsupported. Nothing was sent. Try the built-in demo; there is no desktop fallback.");
- BitmapSource source = BitmapSource.Create(rect.Width, rect.Height, 96, 96, PixelFormats.Bgr32, null, pixels, rect.Width * 4);
- source.Freeze();
- double scale = Math.Min(1, 1600d / Math.Max(rect.Width, rect.Height));
- return scale < 1 ? Resize(source, scale) : source;
- }
- finally
- {
- Array.Clear(pixels);
- if (bits != 0) Marshal.Copy(pixels, 0, bits, pixels.Length);
- if (old != 0) Native.SelectObject(dc, old);
- if (bitmap != 0) Native.DeleteObject(bitmap);
- Native.DeleteDC(dc);
- }
+ // The backend receives only the approved HWND. It must never copy the desktop.
+ var source = FrameCapture.Capture(window, rect, ct);
+ double scale = Math.Min(1, 1280d / Math.Max(rect.Width, rect.Height));
+ return scale < 1 ? Resize(source, scale) : source;
}
private static BitmapSource Resize(BitmapSource source, double scale)
@@ -179,54 +180,181 @@ private static byte[] Encode(BitmapSource source)
return bytes;
}
- private static (ElementInfo[], string, string) ReadAutomation(WindowChoice window, Native.RECT rect, CancellationToken ct)
+ internal static AutomationReadResult ReadAutomation(
+ WindowChoice window, Native.RECT rect, CancellationToken ct, int maxDepth = 18)
{
+ if (maxDepth is < 1 or > 64) throw new ArgumentOutOfRangeException(nameof(maxDepth));
var elements = new List();
var text = new List();
var clock = Stopwatch.StartNew();
int visited = 0, chars = 0;
- string note = "UI Automation names only (not pixel OCR). Password/offscreen subtrees excluded; image is NOT redacted.";
+ bool complete = true, textTruncated = false;
+ string outcome = "complete";
+ string note = "Bounded UI Automation evidence only (not pixel OCR). Password/offscreen subtrees excluded; cross-process descendants are included only beneath the selected HWND root; image is NOT redacted.";
try
{
ct.ThrowIfCancellationRequested();
- var root = AutomationElement.FromHandle(window.Handle);
+ var privacy = AutomationEvidence.CaptureCache();
+ var details = AutomationEvidence.CaptureCache(details: true);
+ var root = AutomationElement.FromHandle(window.Handle).GetUpdatedCache(privacy);
+ if (root.Cached.ProcessId != (int)window.ProcessId)
+ throw new InvalidOperationException("UI Automation root identity changed.");
var walker = TreeWalker.RawViewWalker;
void Walk(AutomationElement node, int depth)
{
ct.ThrowIfCancellationRequested();
- if (++visited > 800 || depth > 18 || text.Count >= 200 || chars >= 12000 || clock.ElapsedMilliseconds > 3000) return;
- var value = node.Current;
+ if (++visited > AutomationEvidence.ScanNodeLimit || depth > maxDepth
+ || clock.ElapsedMilliseconds >= AutomationEvidence.ScanMilliseconds)
+ {
+ complete = false;
+ outcome = depth > maxDepth ? "depth_limit"
+ : visited > AutomationEvidence.ScanNodeLimit ? "node_limit" : "time_limit";
+ return;
+ }
+ var value = node.Cached;
// Do not read Name, Value, TextPattern or descendants of password controls.
- if (value.IsPassword || value.IsOffscreen || value.ProcessId != (int)window.ProcessId) return;
+ // Cross-process descendants are permitted only through this exact HWND-rooted Raw View tree.
+ if (value.IsPassword || value.IsOffscreen) return;
var box = Safety.AutomationBox(value.BoundingRectangle, rect);
if (box is not null)
{
- var name = value.Name?.Trim() ?? "";
- name = name[..Math.Min(name.Length, 256)];
- if (name.Length > 0 && chars + name.Length + 1 <= 12000)
+ node = node.GetUpdatedCache(details);
+ value = node.Cached;
+ if (value.IsPassword || value.IsOffscreen) return;
+ box = Safety.AutomationBox(value.BoundingRectangle, rect);
+ var name = AutomationEvidence.Bounded(value.Name, 256);
+ string automationId = AutomationEvidence.Bounded(value.AutomationId, 128);
+ bool knownMarker = AutomationEvidence.IsKnownAutomationId(automationId);
+ if (box is not null && (name.Length > 0 || knownMarker
+ || value.ControlType == ControlType.Document))
{
+ string label = name.Length > 0 ? name : knownMarker ? automationId : "Document";
string role = value.ControlType.ProgrammaticName.Replace("ControlType.", "").ToLowerInvariant();
- if (value.IsEnabled) elements.Add(new(role, name, box));
- text.Add(name); chars += name.Length + 1;
+ string frameworkId = AutomationEvidence.Bounded(value.FrameworkId, 64);
+ int[]? runtimeId = null;
+ string? toggleState = null;
+ try { runtimeId = node.GetCachedPropertyValue(AutomationElement.RuntimeIdProperty) as int[]; }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or COMException) { }
+ try
+ {
+ if (node.GetCachedPropertyValue(AutomationElement.IsTogglePatternAvailableProperty) is true)
+ toggleState = AutomationEvidence.ToggleState(node);
+ }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or COMException) { }
+ var action = new AutomationEvidence.ActionMetadata(null);
+ try { action = AutomationEvidence.ReadAction(node, cachedPatterns: true); }
+ catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or COMException) { }
+ bool enabled = value.IsEnabled;
+ string? controlId = name.Length > 0 || knownMarker
+ ? AutomationEvidence.ControlId(window, value.ProcessId, runtimeId) : null;
+ elements.Add(new(role, label, box, TargetId: AutomationEvidence.TargetId(window,
+ role, label, box, automationId, frameworkId, value.ProcessId, runtimeId),
+ AutomationId: automationId, FrameworkId: frameworkId,
+ IsEnabled: enabled, Targetable: enabled && controlId is not null, ToggleState: toggleState,
+ HelpText: name.Length > 0 ? AutomationEvidence.Optional(value.HelpText, 256) : null,
+ ItemStatus: name.Length > 0 ? AutomationEvidence.Optional(value.ItemStatus, 128) : null,
+ Action: controlId is null ? null : action.Name, IsReadOnly: action.IsReadOnly,
+ ValueHash: action.ValueHash, ValueLength: action.ValueLength,
+ IsSelected: action.IsSelected, ScrollDirections: action.ScrollDirections,
+ HorizontalScrollPercent: action.HorizontalScrollPercent,
+ VerticalScrollPercent: action.VerticalScrollPercent, ControlId: controlId));
+ if (name.Length > 0)
+ {
+ if (chars + name.Length + 1 <= 12000 && text.Count < 200)
+ {
+ text.Add(name);
+ chars += name.Length + 1;
+ }
+ else textTruncated = true;
+ }
}
}
ct.ThrowIfCancellationRequested();
- if (depth >= 18 || text.Count >= 200 || chars >= 12000 || clock.ElapsedMilliseconds > 3000) return;
- var child = walker.GetFirstChild(node);
- while (child is not null && visited < 800 && text.Count < 200 && chars < 12000 && clock.ElapsedMilliseconds < 3000)
+ if (clock.ElapsedMilliseconds >= AutomationEvidence.ScanMilliseconds)
+ {
+ complete = false;
+ outcome = "time_limit";
+ return;
+ }
+ var child = walker.GetFirstChild(node, privacy);
+ if (depth >= maxDepth)
+ {
+ if (child is not null) { complete = false; outcome = "depth_limit"; }
+ return;
+ }
+ while (child is not null && visited < AutomationEvidence.ScanNodeLimit
+ && clock.ElapsedMilliseconds < AutomationEvidence.ScanMilliseconds)
{
Walk(child, depth + 1);
ct.ThrowIfCancellationRequested();
- if (visited >= 800 || text.Count >= 200 || chars >= 12000 || clock.ElapsedMilliseconds >= 3000) break;
- child = walker.GetNextSibling(child);
+ if (visited >= AutomationEvidence.ScanNodeLimit
+ || clock.ElapsedMilliseconds >= AutomationEvidence.ScanMilliseconds)
+ {
+ complete = false;
+ outcome = visited >= AutomationEvidence.ScanNodeLimit ? "node_limit" : "time_limit";
+ break;
+ }
+ child = walker.GetNextSibling(child, privacy);
}
+ if (child is not null) complete = false;
}
Walk(root, 0);
- if (visited >= 800 || text.Count >= 200 || chars >= 12000 || clock.ElapsedMilliseconds >= 3000) note += " Metadata was bounded/truncated.";
}
catch (OperationCanceledException) { throw; }
catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or COMException or UnauthorizedAccessException)
- { note += " This application exposed incomplete/no accessible text; review carefully."; }
- return (elements.ToArray(), string.Join('\n', text.Distinct()), note);
+ {
+ complete = false;
+ outcome = "provider_error";
+ note += " This application exposed incomplete/no accessible text; review carefully.";
+ }
+ var ambiguousIds = elements.Where(e => e.ControlId is not null)
+ .GroupBy(e => e.ControlId).Where(group => group.Count() > 1).Select(group => group.Key).ToHashSet();
+ var duplicateTargets = elements.GroupBy(e => e.TargetId).Where(group => group.Count() > 1)
+ .Select(group => group.Key).ToHashSet();
+ for (int index = 0; index < elements.Count; index++)
+ if (ambiguousIds.Contains(elements[index].ControlId) || duplicateTargets.Contains(elements[index].TargetId))
+ elements[index] = elements[index] with { ControlId = null, TargetId = null, Action = null, Targetable = false };
+ var result = BoundEvidence(elements, string.Join('\n', text.Distinct()), note, complete, textTruncated);
+ DiagnosticLog.Record("uia_inspection", new
+ {
+ visited, retained = result.Elements.Length, result.Complete, result.ContextTruncated,
+ outcome = complete && !result.Complete ? "control_limit" : outcome,
+ elapsedMs = clock.ElapsedMilliseconds
+ });
+ return result;
+ }
+
+ internal static AutomationReadResult BoundEvidence(IReadOnlyList elements, string text,
+ string note, bool complete, bool textTruncated = false)
+ {
+ static bool Priority(ElementInfo element) => element.Action is not null
+ || element.Role == "document" || AutomationEvidence.IsKnownAutomationId(element.AutomationId);
+ var controls = elements.Where(Priority).ToArray();
+ bool truncated = textTruncated || elements.Count > 200;
+ var retained = controls.Concat(elements.Where(element => !Priority(element))).Take(200).ToArray();
+ complete &= controls.Length <= 200;
+ if (!complete) note += " The controls inspection was incomplete; no automation is authorized.";
+ else if (truncated) note += " Non-action context was shortened; all inspected action controls were retained.";
+ return new(retained, text, note, complete) { ContextTruncated = truncated };
+ }
+}
+
+internal sealed record AutomationReadResult(
+ ElementInfo[] Elements, string Text, string Note, bool Complete)
+{
+ internal bool ContextTruncated { get; init; }
+
+ public ElementInfo[] RequireComplete()
+ {
+ if (!Complete || ContextTruncated) throw new IncompleteAutomationReadException();
+ return Elements;
+ }
+}
+
+internal sealed class IncompleteAutomationReadException : InvalidOperationException
+{
+ public IncompleteAutomationReadException()
+ : base("The controls inspection was incomplete. No camera state or action was accepted. Inspect the selected camera screen again.")
+ {
}
}
\ No newline at end of file
diff --git a/desktop/CaptureTests.cs b/desktop/CaptureTests.cs
index 2ebd9a7..487203b 100644
--- a/desktop/CaptureTests.cs
+++ b/desktop/CaptureTests.cs
@@ -5,6 +5,7 @@
using System.Windows.Automation.Provider;
using System.Windows.Controls;
using System.Windows.Interop;
+using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Threading;
@@ -18,7 +19,7 @@ internal sealed class AssertionFailure(int line) : InvalidOperationException($"C
internal sealed class CaptureFailure(InvalidOperationException error) : InvalidOperationException(error.Message switch
{
"A previous window capture is still returning. Use another application after it finishes, or restart MSGuide." => "provider-busy",
- "Window capture timed out. No snapshot was sent. This application may not support PrintWindow/UI Automation." => "timeout",
+ "Window capture timed out. No snapshot was sent. This application may not support Windows Graphics Capture/UI Automation." => "timeout",
"Selected window disappeared, changed, is minimized, or is not responding. Refresh the chooser." => "window-unavailable",
"Unsupported window dimensions. Resize the selected window and retry." => "unsupported-dimensions",
"PNG exceeds the 2 MB limit; select a smaller window." => "image-too-large",
@@ -26,6 +27,8 @@ internal sealed class AssertionFailure(int line) : InvalidOperationException($"C
"Cannot allocate a window capture context." => "context-allocation",
"Cannot allocate a window bitmap." => "bitmap-allocation",
"This window does not support PrintWindow capture. No desktop fallback is used." => "unsupported",
+ "Windows Graphics Capture is unavailable. No desktop or PrintWindow fallback is used." => "wgc-unavailable",
+ "Windows Graphics Capture did not return a frame. Nothing was sent." => "wgc-frame",
"Capture appears blank, protected, or unsupported. Nothing was sent. Try the built-in demo; there is no desktop fallback." => "blank",
_ => "unclassified"
});
@@ -54,6 +57,45 @@ private static IEnumerable