Synchronize the v1.2.5 branch with the v1.2.7 branch.

.env.example

@@ -10,3 +10,12 @@ MEMMY_GA4_MEASUREMENT_ID=
 # Legal agreement page base URLs
 MEMMY_LEGAL_CN_BASE_URL=https://memmy.cn
 MEMMY_LEGAL_INTL_BASE_URL=https://memmy.bot
+
+# Exact hosts available to sandboxed command plugins. Keep this aligned with
+# the first-party literature-review plugin's declared academic providers.
+MEMMY_COMMAND_PLUGIN_NETWORK_ALLOWLIST=export.arxiv.org,arxiv.org,eutils.ncbi.nlm.nih.gov,pmc.ncbi.nlm.nih.gov,api.openalex.org,api.crossref.org
+
+# Trusted MPP registry. Defaults to the Cloud service, which serves
+# entitlement-gated releases; set this only to point at another registry.
+# HTTPS is required except for a loopback development registry.
+MEMMY_PLUGIN_REGISTRY_URL=

.github/workflows/office-plugin.yml

@@ -0,0 +1,97 @@
+name: Office plugin
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/office-plugin.yml"
+      - "plugins/office/**"
+      - "App/memmy-agent/src/skills/docx/**"
+      - "App/memmy-agent/src/skills/pdf/**"
+      - "App/memmy-agent/src/skills/pptx/**"
+      - "App/memmy-agent/src/skills/xlsx/**"
+      - "App/backend/src/adapters/outbound/plugin-runtime/**"
+      - "App/backend/src/adapters/inbound/local-api/routes/plugin-mcp.ts"
+      - "App/backend/src/tests/office-plugin-e2e.test.ts"
+      - "scripts/test-office-plugin-e2e.mjs"
+      - "App/shell/desktop/electron-builder*.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-26, windows-2025]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+      - name: Provision release-build inputs on macOS
+        if: runner.os == 'macOS'
+        run: |
+          node plugins/office/scripts/provision-macos.mjs
+          brew install poppler
+      - name: Provision release-build inputs on Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          choco install libreoffice-fresh --version=26.2.6 -y --no-progress
+          if ($LASTEXITCODE -ne 0) { throw 'LibreOffice installation failed' }
+          $archive = Join-Path $env:RUNNER_TEMP 'poppler.zip'
+          Invoke-WebRequest 'https://github.com/oschwartz10612/poppler-windows/releases/download/v26.09.0-0/Release-26.09.0-0.zip' -OutFile $archive
+          if ((Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne '7a6f256a0ddf7536182246a5733331bf4677cbcc34f4663774947ad34556c8d0') {
+            # The exact upstream digest is pinned; never continue with an unchecked download.
+            throw 'Poppler archive integrity check failed'
+          }
+          $unpacked = Join-Path $env:RUNNER_TEMP 'poppler'
+          Expand-Archive $archive $unpacked
+          $root = Get-ChildItem $unpacked -Directory | Where-Object { Test-Path (Join-Path $_.FullName 'Library/bin/pdfinfo.exe') } | Select-Object -First 1
+          if (!$root) { throw 'Poppler portable layout not found' }
+          "OFFICE_BUILD_POPPLER=$($root.FullName)" >> $env:GITHUB_ENV
+      - name: Install plugin build dependencies
+        run: npm ci --prefix plugins/office
+      - name: Prepare and smoke-test portable renderer
+        run: node plugins/office/scripts/prepare-runtime.mjs "$RUNNER_TEMP/office-runtime-tree"
+      - name: Install build dependencies
+        run: |
+          npm ci
+          npm install --prefix App/memmy-agent --ignore-scripts --no-audit --no-fund
+          npm run build -w @memmy/local-api-contracts
+          npm run build -w @memmy/knowledge
+          npm run build -w @memmy/agent-source-core
+          npm run build -w @memmy/migrations
+      - name: Build portable runtime archive
+        run: |
+          TARGET=$(node -p 'process.platform + "-" + process.arch')
+          node plugins/office/scripts/package-runtime.mjs --root "$RUNNER_TEMP/office-runtime-tree" --destination plugins/office/release --target "$TARGET" --version "ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" --tools "$RUNNER_TEMP/office-runtime-tree/runtime-tools.json" --set-default
+      - name: Verify runtime and permission boundaries
+        run: |
+          npm --prefix plugins/office run build
+          npm exec -- tsc -p App/backend/tsconfig.json --noEmit
+          if [ "$RUNNER_OS" = macOS ]; then npm --prefix plugins/office test; fi
+      - name: Cold install and run four-format end-to-end validation
+        env:
+          OFFICE_E2E_MANAGED_RUNTIME: "1"
+          MEMMY_PLUGIN_SANDBOX_TRACE: "1"
+          OFFICE_E2E_QA_DIR: ${{ runner.temp }}/office-qa
+          OFFICE_E2E_REPORT: ${{ runner.temp }}/office-e2e.json
+        run: npm run test:office-plugin:e2e
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: office-package-and-validation-${{ matrix.os }}
+          path: |
+            plugins/office/release/
+            ${{ runner.temp }}/office-qa/
+            ${{ runner.temp }}/office-e2e.json

.github/workflows/windows-computer-history-validation.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - ".github/workflows/windows-computer-history-validation.yml"
       - "App/native-computer-history/**"
+      - "App/native-computer-use/**"
       - "App/memmy-agent/src/tools/computer-history/**"
       - "App/memmy-agent/tests/tools/computer-history/**"
       - "App/memmy-agent/scripts/copy-build-assets.mjs"
@@ -24,12 +25,15 @@ jobs:
   validate:
     runs-on: windows-2025
     timeout-minutes: 30
+    env:
+      EXPECTED_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
     defaults:
       run:
         shell: pwsh
     steps:
       - uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
       - uses: actions/setup-node@v4
         with:
@@ -39,7 +43,9 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version: "1.22.12"
-          cache-dependency-path: App/native-computer-history/apps/OpenComputerHistoryWindows/go.mod
+          cache-dependency-path: |
+            App/native-computer-history/apps/OpenComputerHistoryWindows/go.mod
+            App/native-computer-use/apps/OpenComputerUseWindows/go.mod
       - name: Require Windows 11
         run: |
           $ErrorActionPreference = 'Stop'
@@ -51,6 +57,7 @@ jobs:
       - name: Install standalone agent dependencies
         run: npm ci --prefix App/memmy-agent
       - name: Validate native protocol and lifecycle without desktop access
+        id: native_history_tests
         run: |
           $ErrorActionPreference = 'Stop'
           Push-Location App/native-computer-history/apps/OpenComputerHistoryWindows
@@ -62,6 +69,15 @@ jobs:
             go vet -unsafeptr=false ./...
             if ($LASTEXITCODE -ne 0) { throw 'Native recorder vet failed' }
           } finally { Pop-Location }
+      - name: Validate native Computer Use Go module
+        id: native_computer_use_tests
+        run: |
+          $ErrorActionPreference = 'Stop'
+          Push-Location App/native-computer-use/apps/OpenComputerUseWindows
+          try {
+            go test ./...
+            if ($LASTEXITCODE -ne 0) { throw 'Native Computer Use tests failed' }
+          } finally { Pop-Location }
       - name: Build agent and native History recorder from source
         run: npm --prefix App/memmy-agent run build
       - name: Validate packaged app icons with Windows PowerShell 5.1
@@ -155,10 +171,55 @@ jobs:
           node node_modules/vitest/vitest.mjs run
           App/shell/desktop/tests/packaged-runtime-boundary.test.ts
           -t "requires the native Windows 11 History recorder"
+      - name: Emit exact-source Windows provider receipt
+        id: windows_provider_receipt
+        if: ${{ success() && steps.native_history_tests.outcome == 'success' && steps.native_computer_use_tests.outcome == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          if ($env:RUNNER_OS -ne 'Windows') { throw 'Windows provider receipt requires a Windows runner' }
+          $head = (git rev-parse HEAD).Trim()
+          if ($LASTEXITCODE -ne 0 -or $head -ne $env:EXPECTED_SOURCE_SHA) {
+            throw "Executed checkout $head does not match expected source $env:EXPECTED_SOURCE_SHA"
+          }
+          git diff --quiet
+          if ($LASTEXITCODE -ne 0) { throw 'Tracked source changed during validation' }
+          git diff --cached --quiet
+          if ($LASTEXITCODE -ne 0) { throw 'Index changed during validation' }
+          $receipt = [ordered]@{
+            schemaVersion = 1
+            provider = 'windows'
+            status = 'passed'
+            headSha = $head
+            repository = $env:GITHUB_REPOSITORY
+            workflowRef = $env:GITHUB_WORKFLOW_REF
+            workflowSha = $env:GITHUB_WORKFLOW_SHA
+            runId = $env:GITHUB_RUN_ID
+            runAttempt = [int]$env:GITHUB_RUN_ATTEMPT
+            runnerOS = $env:RUNNER_OS
+            runnerArch = $env:RUNNER_ARCH
+            windowsBuild = [Environment]::OSVersion.Version.Build
+            generatedAt = [DateTime]::UtcNow.ToString('o')
+            nativeModules = @(
+              'App/native-computer-history/apps/OpenComputerHistoryWindows'
+              'App/native-computer-use/apps/OpenComputerUseWindows'
+            )
+            liveCaptureValidated = $false
+            signedInstallerValidated = $false
+          }
+          $receipt | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath windows-provider-receipt.json -Encoding utf8NoBOM
+      - name: Upload exact-source Windows provider receipt
+        if: ${{ success() && steps.windows_provider_receipt.outcome == 'success' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-provider-receipt-${{ env.EXPECTED_SOURCE_SHA }}-${{ github.run_attempt }}
+          path: windows-provider-receipt.json
+          if-no-files-found: error
+          retention-days: 14
       - name: Record validation scope
         if: ${{ always() }}
         run: |
           @'
           Validates source-built Windows History recorder identity, PE architecture, exact embedded worker source, pure recorder state/lifecycle self-tests, UTF-8 app catalog output, runtime asset copy, and packaging rules. This does not start live recording or prove real UI Automation/hook behavior.
+          Both native Windows Go modules are tested. The provider receipt is limited to the checks in this job; it does not prove live capture or signed-installer acceptance.
           A signed NSIS installer and a representative real-app capture matrix still require installation testing on a Windows 11 machine.
           '@ | Add-Content -Path $env:GITHUB_STEP_SUMMARY

.gitignore

@@ -14,6 +14,7 @@ memmy-memory-*.tgz
 .DS_Store
 App/memmy-agent/package-lock.json
 sessions/
+artifacts/literature-review-*.json
 .*
 !.github/
 !.github/workflows/
@@ -28,4 +29,8 @@ sessions/
 !App/memmy-agent/src/tools/computer-history/win/.gitkeep
 App/backend/src/adapters/outbound/skill-writer/workspace-bridge/memmy-workspace-bridge.mjs
 Memory/src/agent-source/integration/workspace-bridge/memmy-workspace-bridge.mjs
+
+# Office rendering executables and libraries are provisioned during packaging.
+App/memmy-agent/extra-dependencies/office-rendering/*/bin/
+App/memmy-agent/extra-dependencies/office-rendering/*/native/
 /Mac软件打包/

App/backend/local-api-contracts/src/index.ts

@@ -1,6 +1,7 @@
 /** Memmy local API contract. */
 import { z } from "zod";
 import { ModelThinkingConfigSchema } from "./model-thinking-levels.js";
+import { PluginCapabilityEventPayloadSchema } from "./plugin.js";
 import { normalizeOutputLanguage } from "./output-language.js";
 
 export { normalizeOutputLanguage } from "./output-language.js";
@@ -15,6 +16,7 @@ export * from "./memory-l3-world-model.js";
 export * from "./endpoints.js";
 export * from "./cloud-service.js";
 export * from "./desktop-runtime-manifest.js";
+export * from "./plugin.js";
 export * from "./computer-use-surface.js";
 export * from "./browser-profile-control.js";
 export * from "./embedded-browser-control.js";
@@ -1180,23 +1182,84 @@ export const ModelConfigViewSchema = z.object({
 export type ModelConfigView = z.infer<typeof ModelConfigViewSchema>;
 
 /** Schema for asr transcription input. */
+/**
+ * Largest audio payload a single transcription may carry, in decoded bytes.
+ *
+ * The upstream file-transcription API caps one inline audio item at 20 MiB of
+ * base64, which leaves roughly this much actual audio. An hours-long interview
+ * therefore has to be recorded at a bitrate that fits, because splitting it
+ * restarts speaker numbering and destroys the diarization.
+ */
+export const ASR_MAX_AUDIO_BYTES = 15 * 1024 * 1024;
+
+/** Base64 inflates by 4/3; the route needs headroom for that plus the JSON envelope. */
+export const ASR_MAX_REQUEST_BYTES = Math.ceil(ASR_MAX_AUDIO_BYTES * 4 / 3) + 64 * 1024;
+
 export const AsrTranscriptionInputSchema = z.object({
-    audioBase64: z.string().min(1),
+    audioBase64: z.string().min(1).max(Math.ceil(ASR_MAX_AUDIO_BYTES * 4 / 3)),
     mimeType: z.string().min(1),
-    durationMs: z.number().int().nonnegative().optional()
+    durationMs: z.number().int().nonnegative().optional(),
+    /** Requests speaker separation. Honoured only by upstream models that support it. */
+    diarization: z.boolean().optional()
 });
 export type AsrTranscriptionInput = z.infer<typeof AsrTranscriptionInputSchema>;
 
+/**
+ * One diarized utterance.
+ *
+ * `speakerId` is scoped to a single transcription: the same index in two
+ * transcriptions does not denote the same person.
+ */
+export const AsrTranscriptSegmentSchema = z.object({
+    text: z.string(),
+    speakerId: z.number().int().nonnegative().optional(),
+    startMs: z.number().int().nonnegative().optional(),
+    endMs: z.number().int().nonnegative().optional()
+});
+export type AsrTranscriptSegment = z.infer<typeof AsrTranscriptSegmentSchema>;
+
 /** Schema for asr transcription response. */
 export const AsrTranscriptionResponseSchema = z.object({
     text: z.string(),
     modelId: z.string().trim().min(1),
     provider: CatalogProviderIdSchema,
     source: z.enum(["account", "byok"]),
-    transcribedAt: z.string().datetime()
+    transcribedAt: z.string().datetime(),
+    /** Present only when diarization was requested and the upstream model returned it. */
+    segments: z.array(AsrTranscriptSegmentSchema).optional()
 });
 export type AsrTranscriptionResponse = z.infer<typeof AsrTranscriptionResponseSchema>;
 
+/**
+ * Events on the live ASR stream (`/api/asr/stream`).
+ *
+ * The renderer pushes raw 16 kHz mono PCM as binary frames and receives these
+ * JSON text frames back. `partial` is a provisional reading of the sentence
+ * being spoken; a later frame with the same `sentenceId` replaces it, and
+ * `sentence` is the final text for that id. Speaker attribution is never
+ * present here: the realtime model does not separate speakers, so labels come
+ * only from the diarized pass over the finished recording.
+ */
+export const AsrStreamEventSchema = z.discriminatedUnion("type", [
+    z.object({
+        type: z.enum(["partial", "sentence"]),
+        sentenceId: z.number().int().nonnegative(),
+        text: z.string(),
+        beginMs: z.number().int().nonnegative(),
+        endMs: z.number().int().nonnegative().optional()
+    }),
+    z.object({ type: z.literal("finished") }),
+    z.object({ type: z.literal("error"), message: z.string() })
+]);
+export type AsrStreamEvent = z.infer<typeof AsrStreamEventSchema>;
+
+/** Control frames the renderer sends on the live ASR stream. */
+export const AsrStreamControlSchema = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("start"), sampleRate: z.number().int().positive().optional() }),
+    z.object({ type: z.literal("finish") })
+]);
+export type AsrStreamControl = z.infer<typeof AsrStreamControlSchema>;
+
 export const AccountChannelSchema = z.enum(["email", "phone"]);
 export type AccountChannel = z.infer<typeof AccountChannelSchema>;
 
@@ -1674,11 +1737,29 @@ export const ScanCompletedSseEventSchema = z.object({
     })
 });
 
+export const PluginCapabilitySseEventSchema = z.object({
+    id: z.string(),
+    type: z.literal("plugin.capability_event"),
+    timestamp: z.string().datetime(),
+    payload: PluginCapabilityEventPayloadSchema
+});
+
+export const PluginCapabilitySnapshotSseEventSchema = z.object({
+    id: z.string(),
+    type: z.literal("plugin.capability_snapshot"),
+    timestamp: z.string().datetime(),
+    payload: z.object({
+        calls: z.array(PluginCapabilityEventPayloadSchema.pick({ pluginId: true, capabilityId: true, callId: true, conversationId: true }))
+    })
+});
+
 export const SseEventSchema = z.discriminatedUnion("type", [
     ConnectedSseEventSchema,
     HeartbeatSseEventSchema,
     ScanProgressSseEventSchema,
-    ScanCompletedSseEventSchema
+    ScanCompletedSseEventSchema,
+    PluginCapabilitySseEventSchema,
+    PluginCapabilitySnapshotSseEventSchema
 ]);
 export type SseEvent = z.infer<typeof SseEventSchema>;