diff --git a/.github/workflows/texlive-runtime.yml b/.github/workflows/texlive-runtime.yml new file mode 100644 index 000000000..2a288f711 --- /dev/null +++ b/.github/workflows/texlive-runtime.yml @@ -0,0 +1,104 @@ +name: Managed XeLaTeX runtimes +on: + push: + branches: [codex/managed-texlive-runtime-v1.2.5] + paths: ['.github/workflows/texlive-runtime.yml', 'scripts/texlive/**', 'App/backend/src/adapters/outbound/plugin-runtime/**', 'plugins/office/scripts/package-runtime.mjs'] + workflow_dispatch: +permissions: + contents: read +jobs: + native: + strategy: + fail-fast: false + matrix: + include: + - runner: windows-2025 + target: win32-x64 + - runner: ubuntu-24.04 + target: linux-x64 + - runner: ubuntu-24.04-arm + target: linux-arm64 + - runner: macos-26 + target: darwin-arm64 + - runner: macos-15-intel + target: darwin-x64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 24 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install validation dependencies + run: | + npm ci --ignore-scripts + python -m pip install pymupdf==1.26.5 + - name: Test fixture verification and download pinned OSS bundle + run: | + node --test scripts/texlive/fetch-fixtures.test.mjs + node scripts/texlive/fetch-fixtures.mjs "$RUNNER_TEMP/texlive-brand-fixtures" > "$RUNNER_TEMP/texlive-fixtures-verification.json" + - name: Verify and extract pinned official TinyTeX input + run: node scripts/texlive/provision-runtime.mjs "$RUNNER_TEMP/texlive-input" + - name: Provision review packages (Unix release builder only) + if: runner.os != 'Windows' + run: | + TL="$TEXLIVE_BUILD_SOURCE/$TEXLIVE_BUILD_BIN" + export PATH="$TL:$PATH" + "$TL/tlmgr" option repository https://ctan.math.illinois.edu/systems/texlive/tlnet + "$TL/tlmgr" update --self + "$TL/tlmgr" --no-execute-actions update --all + "$TL/tlmgr" --no-execute-actions install xetex ctex fandol fontspec natbib microtype booktabs tools float hyperref geometry xcolor fancyhdr caption colortbl environ trimspaces psnfss lm tex-gyre + "$TL/mktexlsr" + printf 'y\n' | "$TL/updmap-sys" --syncwithtrees --force + "$TL/fmtutil-sys" --byfmt xelatex + - name: Provision review packages (Windows release builder only) + if: runner.os == 'Windows' + shell: pwsh + run: | + $bin = Join-Path $env:TEXLIVE_BUILD_SOURCE $env:TEXLIVE_BUILD_BIN + $tlmgr = Join-Path $bin 'tlmgr.bat' + $env:PATH = "$bin;$env:PATH" + & $tlmgr option repository https://ctan.math.illinois.edu/systems/texlive/tlnet + if ($LASTEXITCODE -ne 0) { throw 'tlmgr repository failed' } + & $tlmgr update --self + if ($LASTEXITCODE -ne 0) { throw 'tlmgr update failed' } + & $tlmgr --no-execute-actions update --all + if ($LASTEXITCODE -ne 0) { throw 'TeX baseline update failed' } + & $tlmgr --no-execute-actions install xetex ctex fandol fontspec natbib microtype booktabs tools float hyperref geometry xcolor fancyhdr caption colortbl environ trimspaces psnfss lm tex-gyre + if ($LASTEXITCODE -ne 0) { throw 'tlmgr package installation failed' } + & (Join-Path $bin 'mktexlsr.exe') + if ($LASTEXITCODE -ne 0) { throw 'TeX index generation failed' } + 'y' | & (Join-Path $bin 'updmap-sys.exe') --syncwithtrees --force + if ($LASTEXITCODE -ne 0) { throw 'TeX font map generation failed' } + & (Join-Path $bin 'fmtutil-sys.exe') --byfmt xelatex + if ($LASTEXITCODE -ne 0) { throw 'XeLaTeX format generation failed' } + & (Join-Path $bin 'mktexlsr.exe') + if ($LASTEXITCODE -ne 0) { throw 'TeX index generation failed' } + - name: Prepare relocatable distribution and verified archive + run: | + node scripts/texlive/prepare-runtime.mjs --source "$TEXLIVE_BUILD_SOURCE" --destination "$RUNNER_TEMP/texlive-portable" --bin "$TEXLIVE_BUILD_BIN" + node scripts/texlive/package-runtime.mjs --root "$RUNNER_TEMP/texlive-portable" --destination "$RUNNER_TEMP/texlive-release" --target "${{ matrix.target }}" --version 2026.10.28-2 --tools "$RUNNER_TEMP/texlive-portable/tools.json" --set-default + - name: Hide release source and validate managed cold install and offline reuse + run: | + node --input-type=module -e 'import {rename} from "node:fs/promises"; await rename(process.env.TEXLIVE_BUILD_SOURCE,process.env.TEXLIVE_BUILD_SOURCE+"-disabled");' + node --import tsx scripts/texlive/smoke-install.ts "$RUNNER_TEMP/texlive-release" "$RUNNER_TEMP/texlive-installed" "$RUNNER_TEMP/texlive-env.json" "$RUNNER_TEMP/texlive-brand-fixtures" + - name: Verify PDF brand, Chinese text and render every page + run: python scripts/texlive/verify-pdfs.py "$RUNNER_TEMP/texlive-env.json.qa" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: texlive-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/texlive-release/ + ${{ runner.temp }}/texlive-env.json.qa/ + ${{ runner.temp }}/texlive-input/provenance.json + ${{ runner.temp }}/texlive-fixtures-verification.json + retention-days: 14 diff --git a/App/backend/src/adapters/outbound/plugin-runtime/command-adapter.ts b/App/backend/src/adapters/outbound/plugin-runtime/command-adapter.ts index 0994684ee..b39b9e568 100644 --- a/App/backend/src/adapters/outbound/plugin-runtime/command-adapter.ts +++ b/App/backend/src/adapters/outbound/plugin-runtime/command-adapter.ts @@ -4,7 +4,6 @@ import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; import { constants as fsConstants } from "node:fs"; import { access, lstat, mkdir, realpath } from "node:fs/promises"; -import { homedir } from "node:os"; import { dirname, isAbsolute, relative, resolve, sep, win32 } from "node:path"; import { createInterface } from "node:readline"; import { @@ -20,6 +19,7 @@ import { z } from "zod"; import { callTimeoutMs, isCapabilityEvent } from "./shared.js"; import type { PluginAdapter, PluginHostServiceInvoker, PluginRuntimeContext, PluginSession } from "./types.js"; import { createOfficeRuntimeInstaller, type OfficeRuntimeInstaller } from "./office-installer.js"; +import { createTexliveRuntimeInstaller, resolveTexliveRuntime, type TexliveRuntimeInstaller } from "./texlive-installer.js"; import { resolvePluginRegistryBaseUrl } from "../../../config/service-urls.js"; const MAX_OUTPUT_BYTES = 50 * 1024 * 1024; @@ -83,6 +83,7 @@ export interface CreateCommandPluginAdapterOptions { pluginDataRoot?: string; /** Host-created dependency installer; null disables downloading (offline deployments/tests). */ officeRuntimeInstaller?: OfficeRuntimeInstaller | null; + texliveRuntimeInstaller?: TexliveRuntimeInstaller | null; /** Host-owned services callable over the private command runtime protocol. */ hostServices?: PluginHostServiceInvoker; /** Resolves whether the signed-in account holds a manifest-declared entitlement. */ @@ -120,7 +121,8 @@ export function createCommandPluginAdapter(options: CreateCommandPluginAdapterOp options.fileInputRoots ?? [], options.pluginDataRoot, runtimeDependencies.readRoots, - runtimeDependencies.executableRoots + runtimeDependencies.executableRoots, + runtimeDependencies.environment.MEMMY_TEX_CACHE_ROOT ) )); const allowedNetworkHosts = new Set((options.allowedNetworkHosts ?? []).map((host) => host.trim().toLowerCase()).filter(Boolean)); @@ -145,7 +147,8 @@ export function createCommandPluginAdapter(options: CreateCommandPluginAdapterOp code: "plugin_permission_denied" }); const runtimeDependencies = await resolveCommandRuntimeDependencies(config.runtimeDependencies, platform, - options.officeRuntimeInstaller === null ? undefined : options.officeRuntimeInstaller ?? defaultOfficeInstaller(options.pluginDataRoot)); + options.officeRuntimeInstaller === null ? undefined : options.officeRuntimeInstaller ?? defaultOfficeInstaller(options.pluginDataRoot), + options.texliveRuntimeInstaller === null ? undefined : options.texliveRuntimeInstaller ?? defaultTexliveInstaller(options.pluginDataRoot)); const env = resolvePluginEnvironment( config.env, config.secretEnv, @@ -485,11 +488,13 @@ export async function buildPluginSandboxLaunch( fileInputRoots: readonly string[] = [], pluginDataRoot?: string, runtimeReadRoots: readonly string[] = [], - runtimeExecutableRoots: readonly string[] = [] + runtimeExecutableRoots: readonly string[] = [], + runtimeWritableRoot?: string ): Promise { const { root, runtimeCommand, runtimeArgs, cwd, interpreter } = await resolveCommandEntrypoint(context, config); const runtimeRoot = interpreter === "node" ? (platform === "win32" ? dirname(runtimeCommand) : resolve(dirname(runtimeCommand), "..")) : null; const filesystem = await filesystemRules(context, fileInputRoots, pluginDataRoot); + if (runtimeWritableRoot) filesystem.push({ path: await realpath(runtimeWritableRoot), writable: true }); if (platform === "win32") { if (networkEnabled) throw new Error("Windows AppContainer plugins currently require no network capability"); @@ -539,7 +544,8 @@ export interface ResolvedCommandRuntimeDependencies { export async function resolveCommandRuntimeDependencies( dependencies: readonly z.infer[], platform: NodeJS.Platform = process.platform, - officeInstaller?: OfficeRuntimeInstaller + officeInstaller?: OfficeRuntimeInstaller, + texliveInstaller?: TexliveRuntimeInstaller ): Promise { const resolved: ResolvedCommandRuntimeDependencies = { pathEntries: [], readRoots: [], executableRoots: [], environment: {} }; if (dependencies.includes("office")) { @@ -549,21 +555,12 @@ export async function resolveCommandRuntimeDependencies( resolved.executableRoots.push(...office.executableRoots); Object.assign(resolved.environment, office.environment); } - if (!dependencies.includes("texlive")) return resolved; - - if (platform === "darwin") { - const binaryRoot = await realpath("/Library/TeX/texbin").catch(() => null); - const distributionRoot = await realpath("/usr/local/texlive").catch(() => null); - const userTexmfRoot = await realpath(resolve(homedir(), "Library", "texmf")).catch(() => null); - if (binaryRoot) resolved.pathEntries.push(binaryRoot); - if (distributionRoot) { - resolved.readRoots.push(distributionRoot); - resolved.executableRoots.push(distributionRoot); - } - if (userTexmfRoot) { - resolved.readRoots.push(userTexmfRoot); - resolved.environment.TEXMFHOME = userTexmfRoot; - } + if (dependencies.includes("texlive")) { + const tex = await resolveTexliveRuntime(platform, texliveInstaller); + resolved.pathEntries.push(...tex.pathEntries); + resolved.readRoots.push(...tex.readRoots); + resolved.executableRoots.push(...tex.executableRoots); + Object.assign(resolved.environment, tex.environment); } return resolved; } @@ -629,6 +626,16 @@ function defaultOfficeInstaller(pluginDataRoot?: string): OfficeRuntimeInstaller return createOfficeRuntimeInstaller({ cacheRoot: process.env.MEMMY_OFFICE_RUNTIME_CACHE ?? resolve(dirname(pluginDataRoot), "plugin-runtimes", "office"), catalogUrl }); } +function defaultTexliveInstaller(pluginDataRoot?: string): TexliveRuntimeInstaller | undefined { + if (!pluginDataRoot) return undefined; + const registry = resolvePluginRegistryBaseUrl(process.env); + const catalogUrl = process.env.MEMMY_TEXLIVE_RUNTIME_CATALOG_URL + ?? (registry ? new URL("/api/v1/plugins/literature-review/runtimes/{target}", registry).href : undefined); + if (!catalogUrl) return undefined; + return createTexliveRuntimeInstaller({ cacheRoot: process.env.MEMMY_TEXLIVE_RUNTIME_CACHE + ?? resolve(dirname(pluginDataRoot), "plugin-runtimes", "texlive"), catalogUrl }); +} + interface FilesystemRule { path: string; writable: boolean; diff --git a/App/backend/src/adapters/outbound/plugin-runtime/office-installer.ts b/App/backend/src/adapters/outbound/plugin-runtime/office-installer.ts index e332e0153..90428760a 100644 --- a/App/backend/src/adapters/outbound/plugin-runtime/office-installer.ts +++ b/App/backend/src/adapters/outbound/plugin-runtime/office-installer.ts @@ -1,228 +1,19 @@ /** Host-owned, user-scoped installation of verified portable Office runtimes. */ -import { createHash, randomUUID } from "node:crypto"; -import { createReadStream, createWriteStream } from "node:fs"; -import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; -import { dirname, isAbsolute, join, resolve } from "node:path"; -import { Transform } from "node:stream"; -import { pipeline } from "node:stream/promises"; -import { Readable } from "node:stream"; import { z } from "zod"; -import * as yauzl from "yauzl"; - +import { createPortableRuntimeInstaller, extractPortableRuntime, PortableRuntimeManifestSchema, safeRuntimePath, + type PortableRuntimeInstallerOptions, type InstalledPortableRuntime } from "./portable-installer.js"; +export { safeRuntimePath }; export const OFFICE_TOOLS = ["soffice", "pdfinfo", "pdftoppm", "pdftotext"] as const; -const MAX_ARCHIVE = 1024 * 1024 * 1024; -const MAX_EXPANDED = 4 * 1024 * 1024 * 1024; -const MAX_FILES = 50000; -const sha256 = z.string().regex(/^[a-f0-9]{64}$/); -export function safeRuntimePath(value: string): boolean { - return Boolean(value) && value.length < 1000 && !isAbsolute(value) && !/[\\:\0<>"|?*\x01-\x1f]/.test(value) - && value.split("/").every(p => p && p !== "." && p !== ".." && !/[. ]$/.test(p) - && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(p)); -} -const filePath = z.string().refine(safeRuntimePath, "Unsafe runtime path"); -export const OfficeRuntimeManifestSchema = z.object({ +export const OfficeRuntimeManifestSchema = PortableRuntimeManifestSchema.extend({ format: z.literal("memmy-office-runtime-v1"), - target: z.string().regex(/^(darwin|linux|win32)-(x64|arm64)$/), - version: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,80}$/), - archive: z.object({ url: z.string(), sha256, bytes: z.number().int().positive().max(MAX_ARCHIVE) }).strict(), - tools: z.object({ soffice: filePath, pdfinfo: filePath, pdftoppm: filePath, pdftotext: filePath }).strict(), - files: z.record(filePath, sha256).refine(v => Object.keys(v).length > 0 && Object.keys(v).length <= MAX_FILES) -}).strict(); + tools: z.object({ soffice: z.string().refine(safeRuntimePath), pdfinfo: z.string().refine(safeRuntimePath), + pdftoppm: z.string().refine(safeRuntimePath), pdftotext: z.string().refine(safeRuntimePath) }).strict() +}); export type OfficeRuntimeManifest = z.infer; -export interface InstalledOfficeRuntime { root: string; tools: Record; version: string } -export interface OfficeRuntimeInstallerOptions { - cacheRoot: string; - catalogUrl: string; - fetchFn?: typeof fetch; - /** Primarily for progress displays/diagnostics; never contains signed URLs. */ - onProgress?: (stage: string, bytes?: number, total?: number) => void; -} +export type InstalledOfficeRuntime = InstalledPortableRuntime; +export type OfficeRuntimeInstallerOptions = Omit; export interface OfficeRuntimeInstaller { ensure(platform: NodeJS.Platform, arch: string): Promise } -const inFlight = new Map>(); - -function allowedURL(value: string, base?: string): URL { - const url = new URL(value, base); - if (url.username || url.password || !(url.protocol === "https:" || (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)))) throw new Error("Office runtime downloads require HTTPS or loopback HTTP"); - return url; -} -async function hashFile(file: string): Promise { - const hash = createHash("sha256"); - for await (const chunk of createReadStream(file)) hash.update(chunk); - return hash.digest("hex"); -} -async function verifyRoot(root: string, manifest: OfficeRuntimeManifest): Promise { - // Also reject added DLLs/resources that could change native loader behavior. - let entries = 0; - async function scan(directory: string, prefix = ""): Promise { - for (const item of await readdir(directory, { withFileTypes: true })) { - if (++entries > 100000) throw new Error("Office runtime cache has too many entries"); - const name = prefix + item.name; - if (item.isSymbolicLink()) throw new Error("Office runtime cache contains a symbolic link"); - if (item.isDirectory()) await scan(join(directory, item.name), name + "/"); - else if (!item.isFile() || (!manifest.files[name] && name !== ".memmy-runtime.json")) throw new Error(`Unexpected Office runtime cache entry: ${name}`); - } - } - await scan(root); - for (const [name, expected] of Object.entries(manifest.files)) { - const file = resolve(root, name); - const info = await lstat(file); - if (!info.isFile() || info.isSymbolicLink() || await realpath(file) !== file || await hashFile(file) !== expected) throw new Error(`Office runtime file integrity check failed: ${name}`); - } - const tools: Record = {}; - for (const name of OFFICE_TOOLS) { - const file = manifest.tools[name]; - if (!manifest.files[file]) throw new Error(`Office runtime tool is not covered by the manifest: ${name}`); - tools[name] = resolve(root, file); - if (process.platform !== "win32" && !((await lstat(tools[name]!)).mode & 0o111)) throw new Error(`Office runtime tool is not executable: ${name}`); - } - return { root, tools, version: manifest.version }; -} - export function createOfficeRuntimeInstaller(options: OfficeRuntimeInstallerOptions): OfficeRuntimeInstaller { - if (!isAbsolute(options.cacheRoot)) throw new Error("Office cache root must be absolute"); - const fetchFn = options.fetchFn ?? fetch; - const base = allowedURL(options.catalogUrl).href; - return { ensure(platform, arch) { - const target = `${platform}-${arch}`; - if (!/^(darwin|linux|win32)-(x64|arm64)$/.test(target)) return Promise.reject(new Error(`Unsupported Office runtime target: ${target}`)); - const key = `${resolve(options.cacheRoot)}\0${target}\0${base}`; - const running = inFlight.get(key); - if (running) return running; - const result = ensure(target).finally(() => { inFlight.delete(key); }); - inFlight.set(key, result); - return result; - } }; - - async function ensure(target: string): Promise { - await mkdir(options.cacheRoot, { recursive: true }); - const cache = await realpath(options.cacheRoot); - const pointer = join(cache, `${target}.json`); - try { - const record = JSON.parse(await readFile(pointer, "utf8")); - const manifest = OfficeRuntimeManifestSchema.parse(record); - if (manifest.target !== target) throw new Error("Cached runtime target mismatch"); - const installed = await verifyRoot(join(cache, target, manifest.archive.sha256), manifest); - options.onProgress?.("cached"); return installed; - } catch { /* Missing/corrupt caches are repaired by a verified reinstall. */ } - const lock = join(cache, `${target}.lock`); - let acquired = false; - for (let attempt = 0; attempt < 1200; attempt++) { - try { - await mkdir(lock); acquired = true; - await writeFile(join(lock, "owner.json"), JSON.stringify({ pid: process.pid }), { flag: "wx" }); - break; - } - catch (error) { - if (acquired) { await rm(lock, { recursive: true, force: true }); throw error; } - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const info = await lstat(lock).catch(() => null); - if (info?.isSymbolicLink()) throw new Error("Unsafe Office installation lock"); - let abandoned = false; - try { - const owner = JSON.parse(await readFile(join(lock, "owner.json"), "utf8")); - if (!Number.isInteger(owner.pid) || owner.pid <= 0) throw new Error("Invalid installation owner"); - try { process.kill(owner.pid, 0); } - catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") abandoned = true; } - } catch { abandoned = Boolean(info && Date.now() - info.mtimeMs > 30000); } - if (abandoned) await rm(lock, { recursive: true, force: true }); - await new Promise(ok => setTimeout(ok, 500)); - } - } - if (!acquired) throw new Error("Office runtime installation is already in progress; retry shortly"); - let temporary: string | undefined; - try { - // Another Host process may have finished while we waited for the lock. - try { - const cached = OfficeRuntimeManifestSchema.parse(JSON.parse(await readFile(pointer, "utf8"))); - if (cached.target === target) return await verifyRoot(join(cache, target, cached.archive.sha256), cached); - } catch { /* fetch a clean manifest */ } - options.onProgress?.("resolving"); - const url = new URL(base.replace(/\{target\}|%7Btarget%7D/gi, target)); - const response = await fetchFn(url, { redirect: "error", signal: AbortSignal.timeout(30000), headers: { accept: "application/json" } }); - if (!response.ok) throw new Error(`Office runtime catalog returned HTTP ${response.status}`); - const chunks: Buffer[] = []; let manifestBytes = 0; - if (!response.body) throw new Error("Office runtime catalog is empty"); - for await (const chunk of Readable.fromWeb(response.body as never)) { - manifestBytes += chunk.length; chunks.push(Buffer.from(chunk)); - if (manifestBytes > 16 * 1024 * 1024) throw new Error("Office runtime manifest exceeds 16 MiB"); - } - const manifest = OfficeRuntimeManifestSchema.parse(JSON.parse(Buffer.concat(chunks).toString("utf8"))); - if (manifest.target !== target) throw new Error("Office runtime platform/architecture mismatch"); - const paths = Object.keys(manifest.files); - if (paths.some(n => n === ".memmy-runtime.json") || new Set(paths.map(n => n.toLowerCase())).size !== paths.length) throw new Error("Duplicate/reserved Office runtime paths"); - const asset = allowedURL(manifest.archive.url, url.href); - temporary = await mkdtemp(join(cache, ".office-install-")); - const archive = join(temporary, "runtime.zip"), content = join(temporary, "content"); - let lastError: unknown; - for (let retry = 0; retry < 3; retry++) { - try { - options.onProgress?.(retry ? "retrying" : "downloading", 0, manifest.archive.bytes); - const download = await fetchFn(asset, { redirect: "error", signal: AbortSignal.timeout(10 * 60 * 1000) }); - if (!download.ok || !download.body) throw new Error(`Office runtime download returned HTTP ${download.status}`); - let received = 0; const digest = createHash("sha256"); - const meter = new Transform({ transform(chunk: Buffer, _encoding, callback) { - received += chunk.length; - if (received > manifest.archive.bytes || received > MAX_ARCHIVE) { callback(new Error("Office runtime download exceeds declared size")); return; } - digest.update(chunk); options.onProgress?.("downloading", received, manifest.archive.bytes); callback(null, chunk); - } }); - await pipeline(Readable.fromWeb(download.body as never), meter, createWriteStream(archive, { flags: "w", mode: 0o600 })); - if (received !== manifest.archive.bytes || digest.digest("hex") !== manifest.archive.sha256) throw new Error("Office runtime archive integrity check failed"); - lastError = undefined; break; - } catch (error) { lastError = error; await rm(archive, { force: true }); } - } - if (lastError) throw lastError; - options.onProgress?.("extracting"); - await mkdir(content); - await extractOfficeRuntime(archive, content, manifest); - await verifyRoot(content, manifest); - await writeFile(join(content, ".memmy-runtime.json"), JSON.stringify(manifest)); - const parent = join(cache, target); await mkdir(parent, { recursive: true }); - if (await realpath(parent) !== parent) throw new Error("Unsafe Office cache directory"); - const destination = join(parent, manifest.archive.sha256); - // The lock excludes readers/installers. Replace only this corrupt content-addressed cache. - await rm(destination, { recursive: true, force: true }); - await rename(content, destination); - const pending = join(cache, `${target}.${randomUUID()}.tmp`); - await writeFile(pending, JSON.stringify(manifest), { flag: "wx" }); await rename(pending, pointer); - options.onProgress?.("installed"); - return await verifyRoot(destination, manifest); - } finally { - if (temporary) await rm(temporary, { recursive: true, force: true }); - await rm(lock, { recursive: true, force: true }); - } - } -} - -export async function extractOfficeRuntime(archive: string, root: string, manifest: OfficeRuntimeManifest): Promise { - const zip = await new Promise((ok, reject) => yauzl.open(archive, { lazyEntries: true, autoClose: true, validateEntrySizes: true }, (error, file) => error ? reject(error) : ok(file!))); - const seen = new Set(); let expanded = 0, files = 0, entries = 0; - await new Promise((ok, reject) => { - let failed = false; - const abort = (error: unknown) => { if (failed) return; failed = true; zip.close(); reject(error); }; - zip.on("error", abort); zip.once("end", () => { if (!failed) ok(); }); - zip.on("entry", (entry: yauzl.Entry) => { - void (async () => { - if (++entries > 100000) throw new Error("Office runtime ZIP has too many entries"); - const name = entry.fileName.replace(/\/$/, ""); - if (!safeRuntimePath(name) || ((entry.externalFileAttributes >>> 16) & 0o170000) === 0o120000) throw new Error("Unsafe Office runtime ZIP entry"); - if (entry.fileName.endsWith("/")) { zip.readEntry(); return; } - if (++files > MAX_FILES || seen.has(name.toLowerCase()) || !manifest.files[name] || entry.uncompressedSize > MAX_EXPANDED - expanded) throw new Error("Unexpected/oversized Office runtime ZIP entry"); - seen.add(name.toLowerCase()); expanded += entry.uncompressedSize; - const target = resolve(root, name); await mkdir(dirname(target), { recursive: true }); - const stream = await new Promise((yes, no) => zip.openReadStream(entry, (error, value) => error ? no(error) : yes(value!))); - let size = 0; const hash = createHash("sha256"); - const check = new Transform({ transform(chunk: Buffer, _encoding, cb) { - size += chunk.length; - if (size > entry.uncompressedSize || size > MAX_EXPANDED) { cb(new Error("Office runtime ZIP expansion limit exceeded")); return; } - hash.update(chunk); cb(null, chunk); - } }); - await pipeline(stream, check, createWriteStream(target, { flags: "wx", mode: 0o600 })); - if (size !== entry.uncompressedSize || hash.digest("hex") !== manifest.files[name]) throw new Error(`Office runtime file digest mismatch: ${name}`); - await chmod(target, ((entry.externalFileAttributes >>> 16) & 0o111) ? 0o755 : 0o644); - zip.readEntry(); - })().catch(abort); - }); - zip.readEntry(); - }); - if (seen.size !== Object.keys(manifest.files).length) throw new Error("Office runtime archive is incomplete"); + return createPortableRuntimeInstaller({ ...options, kind: "office", requiredTools: OFFICE_TOOLS }); } +export const extractOfficeRuntime = extractPortableRuntime; diff --git a/App/backend/src/adapters/outbound/plugin-runtime/portable-installer.ts b/App/backend/src/adapters/outbound/plugin-runtime/portable-installer.ts new file mode 100644 index 000000000..75d510a02 --- /dev/null +++ b/App/backend/src/adapters/outbound/plugin-runtime/portable-installer.ts @@ -0,0 +1,237 @@ +/** Host-owned, user-scoped installation of verified portable runtimes. */ +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { Readable } from "node:stream"; +import { z } from "zod"; +import * as yauzl from "yauzl"; + +const MAX_ARCHIVE = 1024 * 1024 * 1024; +const MAX_EXPANDED = 4 * 1024 * 1024 * 1024; +const MAX_FILES = 50000; +const sha256 = z.string().regex(/^[a-f0-9]{64}$/); +export function safeRuntimePath(value: string): boolean { + return Boolean(value) && value.length < 1000 && !isAbsolute(value) && !/[\\:\0<>"|?*\x01-\x1f]/.test(value) + && value.split("/").every(p => p && p !== "." && p !== ".." && !/[. ]$/.test(p) + && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(p)); +} +const filePath = z.string().refine(safeRuntimePath, "Unsafe runtime path"); +export const PortableRuntimeManifestSchema = z.object({ + format: z.string().regex(/^memmy-(office|texlive)-runtime-v1$/), + target: z.string().regex(/^(darwin|linux|win32)-(x64|arm64)$/), + version: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,80}$/), + archive: z.object({ url: z.string(), sha256, bytes: z.number().int().positive().max(MAX_ARCHIVE) }).strict(), + tools: z.record(z.string().regex(/^[a-z][a-z0-9-]*$/), filePath).refine(v => Object.keys(v).length > 0 && Object.keys(v).length <= 16), + files: z.record(filePath, sha256).refine(v => Object.keys(v).length > 0 && Object.keys(v).length <= MAX_FILES) +}).strict(); +export type PortableRuntimeManifest = z.infer; +export interface InstalledPortableRuntime { root: string; tools: Record; version: string } +export interface PortableRuntimeInstallerOptions { + kind: "office" | "texlive"; + requiredTools: readonly string[]; + cacheRoot: string; + catalogUrl: string; + fetchFn?: typeof fetch; + /** Primarily for progress displays/diagnostics; never contains signed URLs. */ + onProgress?: (stage: string, bytes?: number, total?: number) => void; +} +export interface PortableRuntimeInstaller { ensure(platform: NodeJS.Platform, arch: string): Promise } +const inFlight = new Map>(); + +function allowedURL(value: string, base?: string): URL { + const url = new URL(value, base); + if (url.username || url.password || !(url.protocol === "https:" || (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)))) throw new Error("Portable runtime downloads require HTTPS or loopback HTTP"); + return url; +} +async function hashFile(file: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(file)) hash.update(chunk); + return hash.digest("hex"); +} +async function verifyRoot(root: string, manifest: PortableRuntimeManifest): Promise { + // Also reject added DLLs/resources that could change native loader behavior. + let entries = 0; + async function scan(directory: string, prefix = ""): Promise { + for (const item of await readdir(directory, { withFileTypes: true })) { + if (++entries > 100000) throw new Error("Portable runtime cache has too many entries"); + const name = prefix + item.name; + if (item.isSymbolicLink()) throw new Error("Portable runtime cache contains a symbolic link"); + if (item.isDirectory()) await scan(join(directory, item.name), name + "/"); + else if (!item.isFile() || (!manifest.files[name] && name !== ".memmy-runtime.json")) throw new Error(`Unexpected Portable runtime cache entry: ${name}`); + } + } + await scan(root); + for (const [name, expected] of Object.entries(manifest.files)) { + const file = resolve(root, name); + const info = await lstat(file); + if (!info.isFile() || info.isSymbolicLink() || await realpath(file) !== file || await hashFile(file) !== expected) throw new Error(`Portable runtime file integrity check failed: ${name}`); + } + const tools: Record = {}; + for (const name of Object.keys(manifest.tools)) { + const file = manifest.tools[name]!; + if (!manifest.files[file]) throw new Error(`Portable runtime tool is not covered by the manifest: ${name}`); + tools[name] = resolve(root, file); + if (process.platform !== "win32" && !((await lstat(tools[name]!)).mode & 0o111)) throw new Error(`Portable runtime tool is not executable: ${name}`); + } + return { root, tools, version: manifest.version }; +} + +export function createPortableRuntimeInstaller(options: PortableRuntimeInstallerOptions): PortableRuntimeInstaller { + if (!isAbsolute(options.cacheRoot)) throw new Error("Portable cache root must be absolute"); + const fetchFn = options.fetchFn ?? fetch; + const base = allowedURL(options.catalogUrl).href; + function parseManifest(record: unknown): PortableRuntimeManifest { + const manifest = PortableRuntimeManifestSchema.parse(record); + if (manifest.format !== `memmy-${options.kind}-runtime-v1` + || Object.keys(manifest.tools).sort().join(",") !== [...options.requiredTools].sort().join(",")) { + throw new Error("Portable runtime kind/tools mismatch"); + } + return manifest; + } + return { ensure(platform, arch) { + const target = `${platform}-${arch}`; + if (!/^(darwin|linux|win32)-(x64|arm64)$/.test(target)) return Promise.reject(new Error(`Unsupported Portable runtime target: ${target}`)); + const key = `${resolve(options.cacheRoot)}\0${options.kind}\0${target}\0${base}`; + const running = inFlight.get(key); + if (running) return running; + const result = ensure(target).finally(() => { inFlight.delete(key); }); + inFlight.set(key, result); + return result; + } }; + + async function ensure(target: string): Promise { + await mkdir(options.cacheRoot, { recursive: true }); + const cache = await realpath(options.cacheRoot); + const pointer = join(cache, `${target}.json`); + try { + const record = JSON.parse(await readFile(pointer, "utf8")); + const manifest = parseManifest(record); + if (manifest.target !== target) throw new Error("Cached runtime target mismatch"); + const installed = await verifyRoot(join(cache, target, manifest.archive.sha256), manifest); + options.onProgress?.("cached"); return installed; + } catch { /* Missing/corrupt caches are repaired by a verified reinstall. */ } + const lock = join(cache, `${target}.lock`); + let acquired = false; + for (let attempt = 0; attempt < 1200; attempt++) { + try { + await mkdir(lock); acquired = true; + await writeFile(join(lock, "owner.json"), JSON.stringify({ pid: process.pid }), { flag: "wx" }); + break; + } + catch (error) { + if (acquired) { await rm(lock, { recursive: true, force: true }); throw error; } + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const info = await lstat(lock).catch(() => null); + if (info?.isSymbolicLink()) throw new Error("Unsafe Portable installation lock"); + let abandoned = false; + try { + const owner = JSON.parse(await readFile(join(lock, "owner.json"), "utf8")); + if (!Number.isInteger(owner.pid) || owner.pid <= 0) throw new Error("Invalid installation owner"); + try { process.kill(owner.pid, 0); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") abandoned = true; } + } catch { abandoned = Boolean(info && Date.now() - info.mtimeMs > 30000); } + if (abandoned) await rm(lock, { recursive: true, force: true }); + await new Promise(ok => setTimeout(ok, 500)); + } + } + if (!acquired) throw new Error("Portable runtime installation is already in progress; retry shortly"); + let temporary: string | undefined; + try { + // Another Host process may have finished while we waited for the lock. + try { + const cached = parseManifest(JSON.parse(await readFile(pointer, "utf8"))); + if (cached.target === target) return await verifyRoot(join(cache, target, cached.archive.sha256), cached); + } catch { /* fetch a clean manifest */ } + options.onProgress?.("resolving"); + const url = new URL(base.replace(/\{target\}|%7Btarget%7D/gi, target)); + const response = await fetchFn(url, { redirect: "error", signal: AbortSignal.timeout(30000), headers: { accept: "application/json" } }); + if (!response.ok) throw new Error(`Portable runtime catalog returned HTTP ${response.status}`); + const chunks: Buffer[] = []; let manifestBytes = 0; + if (!response.body) throw new Error("Portable runtime catalog is empty"); + for await (const chunk of Readable.fromWeb(response.body as never)) { + manifestBytes += chunk.length; chunks.push(Buffer.from(chunk)); + if (manifestBytes > 16 * 1024 * 1024) throw new Error("Portable runtime manifest exceeds 16 MiB"); + } + const manifest = parseManifest(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + if (manifest.target !== target) throw new Error("Portable runtime platform/architecture mismatch"); + const paths = Object.keys(manifest.files); + if (paths.some(n => n === ".memmy-runtime.json") || new Set(paths.map(n => n.toLowerCase())).size !== paths.length) throw new Error("Duplicate/reserved Portable runtime paths"); + const asset = allowedURL(manifest.archive.url, url.href); + temporary = await mkdtemp(join(cache, ".portable-install-")); + const archive = join(temporary, "runtime.zip"), content = join(temporary, "content"); + let lastError: unknown; + for (let retry = 0; retry < 3; retry++) { + try { + options.onProgress?.(retry ? "retrying" : "downloading", 0, manifest.archive.bytes); + const download = await fetchFn(asset, { redirect: "error", signal: AbortSignal.timeout(10 * 60 * 1000) }); + if (!download.ok || !download.body) throw new Error(`Portable runtime download returned HTTP ${download.status}`); + let received = 0; const digest = createHash("sha256"); + const meter = new Transform({ transform(chunk: Buffer, _encoding, callback) { + received += chunk.length; + if (received > manifest.archive.bytes || received > MAX_ARCHIVE) { callback(new Error("Portable runtime download exceeds declared size")); return; } + digest.update(chunk); options.onProgress?.("downloading", received, manifest.archive.bytes); callback(null, chunk); + } }); + await pipeline(Readable.fromWeb(download.body as never), meter, createWriteStream(archive, { flags: "w", mode: 0o600 })); + if (received !== manifest.archive.bytes || digest.digest("hex") !== manifest.archive.sha256) throw new Error("Portable runtime archive integrity check failed"); + lastError = undefined; break; + } catch (error) { lastError = error; await rm(archive, { force: true }); } + } + if (lastError) throw lastError; + options.onProgress?.("extracting"); + await mkdir(content); + await extractPortableRuntime(archive, content, manifest); + await verifyRoot(content, manifest); + await writeFile(join(content, ".memmy-runtime.json"), JSON.stringify(manifest)); + const parent = join(cache, target); await mkdir(parent, { recursive: true }); + if (await realpath(parent) !== parent) throw new Error("Unsafe Portable cache directory"); + const destination = join(parent, manifest.archive.sha256); + // The lock excludes readers/installers. Replace only this corrupt content-addressed cache. + await rm(destination, { recursive: true, force: true }); + await rename(content, destination); + const pending = join(cache, `${target}.${randomUUID()}.tmp`); + await writeFile(pending, JSON.stringify(manifest), { flag: "wx" }); await rename(pending, pointer); + options.onProgress?.("installed"); + return await verifyRoot(destination, manifest); + } finally { + if (temporary) await rm(temporary, { recursive: true, force: true }); + await rm(lock, { recursive: true, force: true }); + } + } +} + +export async function extractPortableRuntime(archive: string, root: string, manifest: PortableRuntimeManifest): Promise { + const zip = await new Promise((ok, reject) => yauzl.open(archive, { lazyEntries: true, autoClose: true, validateEntrySizes: true }, (error, file) => error ? reject(error) : ok(file!))); + const seen = new Set(); let expanded = 0, files = 0, entries = 0; + await new Promise((ok, reject) => { + let failed = false; + const abort = (error: unknown) => { if (failed) return; failed = true; zip.close(); reject(error); }; + zip.on("error", abort); zip.once("end", () => { if (!failed) ok(); }); + zip.on("entry", (entry: yauzl.Entry) => { + void (async () => { + if (++entries > 100000) throw new Error("Portable runtime ZIP has too many entries"); + const name = entry.fileName.replace(/\/$/, ""); + if (!safeRuntimePath(name) || ((entry.externalFileAttributes >>> 16) & 0o170000) === 0o120000) throw new Error("Unsafe Portable runtime ZIP entry"); + if (entry.fileName.endsWith("/")) { zip.readEntry(); return; } + if (++files > MAX_FILES || seen.has(name.toLowerCase()) || !manifest.files[name] || entry.uncompressedSize > MAX_EXPANDED - expanded) throw new Error("Unexpected/oversized Portable runtime ZIP entry"); + seen.add(name.toLowerCase()); expanded += entry.uncompressedSize; + const target = resolve(root, name); await mkdir(dirname(target), { recursive: true }); + const stream = await new Promise((yes, no) => zip.openReadStream(entry, (error, value) => error ? no(error) : yes(value!))); + let size = 0; const hash = createHash("sha256"); + const check = new Transform({ transform(chunk: Buffer, _encoding, cb) { + size += chunk.length; + if (size > entry.uncompressedSize || size > MAX_EXPANDED) { cb(new Error("Portable runtime ZIP expansion limit exceeded")); return; } + hash.update(chunk); cb(null, chunk); + } }); + await pipeline(stream, check, createWriteStream(target, { flags: "wx", mode: 0o600 })); + if (size !== entry.uncompressedSize || hash.digest("hex") !== manifest.files[name]) throw new Error(`Portable runtime file digest mismatch: ${name}`); + await chmod(target, ((entry.externalFileAttributes >>> 16) & 0o111) ? 0o755 : 0o644); + zip.readEntry(); + })().catch(abort); + }); + zip.readEntry(); + }); + if (seen.size !== Object.keys(manifest.files).length) throw new Error("Portable runtime archive is incomplete"); +} diff --git a/App/backend/src/adapters/outbound/plugin-runtime/tests/command-adapter.test.ts b/App/backend/src/adapters/outbound/plugin-runtime/tests/command-adapter.test.ts index 3e735a383..4558cf6ab 100644 --- a/App/backend/src/adapters/outbound/plugin-runtime/tests/command-adapter.test.ts +++ b/App/backend/src/adapters/outbound/plugin-runtime/tests/command-adapter.test.ts @@ -483,6 +483,18 @@ describe("CommandPluginAdapter", () => { expect(String(fetchFn.mock.calls[0]?.[0])).toBe(`http://127.0.0.1:19391/api/v1/plugins/office/runtimes/win32-${process.arch}`); }); + it("automatically resolves the TeX runtime from the literature registry when missing", async () => { + const pluginContext = context(); + vi.stubEnv("MEMMY_PLUGIN_REGISTRY_URL", "http://127.0.0.1:19391/api"); + vi.stubEnv("MEMMY_TEXLIVE_RUNTIME_PREFER_MANAGED", "1"); + vi.stubEnv("MEMMY_TEXLIVE_RUNTIME_CATALOG_URL", ""); + delete process.env.MEMMY_TEXLIVE_RUNTIME_CATALOG_URL; + const fetchFn = vi.fn(async (_input: unknown) => new Response("not provisioned", { status: 404 })); vi.stubGlobal("fetch", fetchFn); + pluginContext.plugin.manifest.runtime.config = { ...pluginContext.plugin.manifest.runtime.config, runtimeDependencies: ["texlive"] }; + await expect(createCommandPluginAdapter({ platform: "win32", pluginDataRoot: join(root!, "plugin-data") }).activate(pluginContext)).rejects.toThrow(/HTTP 404/); + expect(String(fetchFn.mock.calls[0]?.[0])).toBe(`http://127.0.0.1:19391/api/v1/plugins/literature-review/runtimes/win32-${process.arch}`); + }); + it("keeps plugin CLR/PowerShell variables out of the privileged Windows launcher", async () => { const pluginContext = context(); pluginContext.plugin.manifest.runtime.config = { ...pluginContext.plugin.manifest.runtime.config, interpreter: "node", env: { COR_ENABLE_PROFILING: "1", COR_PROFILER_PATH: "untrusted.dll", PSModulePath: "untrusted-modules", SystemRoot: "untrusted-root", LOCALAPPDATA: "untrusted-profile" } }; diff --git a/App/backend/src/adapters/outbound/plugin-runtime/tests/texlive-installer.test.ts b/App/backend/src/adapters/outbound/plugin-runtime/tests/texlive-installer.test.ts new file mode 100644 index 000000000..842a07044 --- /dev/null +++ b/App/backend/src/adapters/outbound/plugin-runtime/tests/texlive-installer.test.ts @@ -0,0 +1,73 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import JSZip from "jszip"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createTexliveRuntimeInstaller, resolveTexliveRuntime } from "../texlive-installer.js"; + +const roots: string[] = []; +afterEach(async () => { vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const hash = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); +async function fixture() { + const cacheRoot = await mkdtemp(join(tmpdir(), "memmy-texlive-")); roots.push(cacheRoot); + const zip = new JSZip(), files: Record = {}, tools: Record = {}; + for (const name of ["xelatex", "kpsewhich"]) { + const file = `bin/${name}.exe`, content = `fixture-${name}`; + zip.file(file, content, { unixPermissions: 0o100755 }); files[file] = hash(content); tools[name] = file; + } + zip.file("texmf-dist/fonts/FandolSong-Regular.otf", "font"); files["texmf-dist/fonts/FandolSong-Regular.otf"] = hash("font"); + const bytes = await zip.generateAsync({ type: "nodebuffer", platform: "UNIX" }); + const manifest = { format: "memmy-texlive-runtime-v1", target: "win32-x64", version: "review-1", tools, files, + archive: { url: "/texlive.zip", sha256: hash(bytes), bytes: bytes.length } }; + let downloads = 0, offline = false; + const installer = createTexliveRuntimeInstaller({ cacheRoot, catalogUrl: "https://registry.example/texlive/{target}", + fetchFn: (async input => { + if (offline) throw new Error("offline"); + if (String(input).endsWith("win32-x64")) return Response.json(manifest); + downloads++; return new Response(bytes); + }) as typeof fetch }); + return { installer, manifest, cacheRoot, downloads: () => downloads, offline: () => { offline = true; } }; +} +describe("managed review XeLaTeX", () => { + it("downloads a verified font/engine bundle once and reuses it offline", async () => { + const f = await fixture(); + const [a, b] = await Promise.all([f.installer.ensure("win32", "x64"), f.installer.ensure("win32", "x64")]); + expect(a).toEqual(b); expect(f.downloads()).toBe(1); + f.offline(); expect(await f.installer.ensure("win32", "x64")).toEqual(a); + }); + it("rejects an Office descriptor on the TeX route", async () => { + const f = await fixture(); f.manifest.format = "memmy-office-runtime-v1"; + await expect(f.installer.ensure("win32", "x64")).rejects.toThrow(/kind\/tools/); + }); + it("repairs a changed font as well as changed executables", async () => { + const f = await fixture(), installed = await f.installer.ensure("win32", "x64"); + await writeFile(join(installed.root, "texmf-dist/fonts/FandolSong-Regular.otf"), "bad-font"); + await f.installer.ensure("win32", "x64"); expect(f.downloads()).toBe(2); + }); + it("mounts only verified runtime roots and keeps writable TeX caches outside them", async () => { + vi.stubEnv("MEMMY_TEXLIVE_RUNTIME_PREFER_MANAGED", "1"); + const f = await fixture(), installed = await f.installer.ensure("win32", "x64"); + const ensure = vi.fn(async () => installed), probe = vi.fn(async () => true); + const resolved = await resolveTexliveRuntime("win32", { ensure }, probe); + expect(resolved.readRoots).toEqual([installed.root]); + expect(resolved.executableRoots).toEqual([installed.root]); + expect(resolved.pathEntries).toEqual([dirname(installed.tools.xelatex!)]); + expect(resolved.environment.TEXMFVAR).not.toContain(installed.root); + expect(resolved.environment.MEMMY_TEX_FONTSET).toBe("fandol"); + // A font cache write does not invalidate the runtime integrity manifest. + await writeFile(join(resolved.environment.MEMMY_TEX_CACHE_ROOT!, "font-cache"), "cached"); + expect(await f.installer.ensure("win32", "x64")).toEqual(installed); + expect(f.downloads()).toBe(1); + }); + it("refuses an incomplete installation rather than presenting it as ready", async () => { + vi.stubEnv("MEMMY_TEXLIVE_RUNTIME_PREFER_MANAGED", "1"); + const f = await fixture(), installed = await f.installer.ensure("win32", "x64"); + await expect(resolveTexliveRuntime("win32", { ensure: async () => installed }, async () => false)).rejects.toThrow(/incomplete/); + }); + it("reports a failed download instead of returning an unverified binary", async () => { + const f = await fixture(); f.offline(); + await expect(f.installer.ensure("win32", "x64")).rejects.toThrow("offline"); + await expect(readFile(join(f.cacheRoot, "win32-x64.json"))).rejects.toThrow(); + }); +}); diff --git a/App/backend/src/adapters/outbound/plugin-runtime/texlive-installer.ts b/App/backend/src/adapters/outbound/plugin-runtime/texlive-installer.ts new file mode 100644 index 000000000..d5a4127e0 --- /dev/null +++ b/App/backend/src/adapters/outbound/plugin-runtime/texlive-installer.ts @@ -0,0 +1,90 @@ +/** Host-owned portable XeLaTeX dependency. It never changes system TeX or PATH. */ +import { execFile } from "node:child_process"; +import { access, mkdir, realpath } from "node:fs/promises"; +import { homedir } from "node:os"; +import { dirname, join, delimiter, parse } from "node:path"; +import { promisify } from "node:util"; +import { createPortableRuntimeInstaller, type PortableRuntimeInstallerOptions, type PortableRuntimeInstaller } from "./portable-installer.js"; + +const run = promisify(execFile); +export type TexliveRuntimeInstaller = PortableRuntimeInstaller; +export const TEXLIVE_TOOLS = ["xelatex", "kpsewhich"] as const; +// Includes Chinese fonts and the dependencies of every bundled review template. +export const REVIEW_TEX_FILES = ["ctex.sty", "fontspec.sty", "natbib.sty", "microtype.sty", "booktabs.sty", "graphicx.sty", + "tabularx.sty", "float.sty", "hyperref.sty", "geometry.sty", "xcolor.sty", "fancyhdr.sty", "caption.sty", "colortbl.sty", + "environ.sty", "trimspaces.sty", "times.sty", "FandolSong-Regular.otf", "FandolHei-Regular.otf", "FandolFang-Regular.otf", + "FandolKai-Regular.otf", "lmroman10-regular.otf", "lmsans10-regular.otf"] as const; +export function createTexliveRuntimeInstaller(options: Omit): TexliveRuntimeInstaller { + return createPortableRuntimeInstaller({ ...options, kind: "texlive", requiredTools: TEXLIVE_TOOLS }); +} +export interface ResolvedTexliveRuntime { + pathEntries: string[]; readRoots: string[]; executableRoots: string[]; environment: Record; +} +export type TexliveProbe = (engine: string, kpsewhich: string, environment: Record) => Promise; +export const probeTexlive: TexliveProbe = async (engine, kpsewhich, environment) => { + try { + const env = { ...process.env, ...environment }; + await run(engine, ["--version"], { env, timeout: 10000, maxBuffer: 65536, windowsHide: true }); + const { stdout } = await run(kpsewhich, [...REVIEW_TEX_FILES], { env, timeout: 15000, maxBuffer: 65536, windowsHide: true }); + return stdout.trim().split(/\r?\n/u).length === REVIEW_TEX_FILES.length; + } catch { return false; } +}; +export async function resolveTexliveRuntime(platform: NodeJS.Platform, installer?: TexliveRuntimeInstaller, + probe: TexliveProbe = probeTexlive): Promise { + const result: ResolvedTexliveRuntime = { pathEntries: [], readRoots: [], executableRoots: [], environment: {} }; + const extension = platform === "win32" ? ".exe" : ""; + if (process.env.MEMMY_TEXLIVE_RUNTIME_PREFER_MANAGED !== "1") { + const binaryRoot = platform === "darwin" ? await realpath("/Library/TeX/texbin").catch(() => null) : null; + const find = async (name: string): Promise => { + for (const directory of [binaryRoot, ...(process.env.PATH ?? "").split(delimiter)]) { + if (!directory) continue; + const file = join(directory, name + extension); + try { await access(file); return file; } catch { /* continue */ } + } + return null; + }; + const engine = await find("xelatex"), kpsewhich = await find("kpsewhich"); + if (engine && kpsewhich && await probe(engine, kpsewhich, {})) { + result.pathEntries.push(dirname(engine), dirname(kpsewhich)); + if (binaryRoot) result.pathEntries.push(binaryRoot); + if (platform === "darwin") { + for (const candidate of ["/usr/local/texlive", join(homedir(), "Library", "texmf")]) { + const root = await realpath(candidate).catch(() => null); + if (root) { result.readRoots.push(root); result.executableRoots.push(root); + if (candidate.endsWith("texmf")) result.environment.TEXMFHOME = root; } + } + } + if (platform !== "darwin") { + // Query the chosen distribution instead of granting an arbitrary parent directory. + try { + const { stdout } = await run(kpsewhich, ["--var-value=TEXMFDIST"], { timeout: 10000, windowsHide: true }); + const root = await realpath(dirname(stdout.trim())); + if (root === parse(root).root) throw new Error("Invalid TeX distribution root"); + result.readRoots.push(root); result.executableRoots.push(root); + } catch { return resolveTexliveRuntimeManaged(platform, installer, probe); } + } + result.environment.MEMMY_TEX_FONTSET = "fandol"; + return result; + } + } + return resolveTexliveRuntimeManaged(platform, installer, probe); +} +async function resolveTexliveRuntimeManaged(platform: NodeJS.Platform, installer: TexliveRuntimeInstaller | undefined, + probe: TexliveProbe): Promise { + const result: ResolvedTexliveRuntime = { pathEntries: [], readRoots: [], executableRoots: [], environment: {} }; + if (!installer) return result; // Offline/test hosts still report the missing engine at render preflight. + const installed = await installer.ensure(platform, process.arch); + // TeX caches must be separate from the immutable, hash-verified installation. + const mutableRoot = join(dirname(installed.root), "texmf-cache", installed.version); + await mkdir(mutableRoot, { recursive: true }); + result.environment = { TEXMFVAR: join(mutableRoot, "var"), TEXMFCONFIG: join(mutableRoot, "config"), + TEXMFHOME: join(mutableRoot, "home"), MEMMY_TEX_FONTSET: "fandol" }; + if (!await probe(installed.tools.xelatex!, installed.tools.kpsewhich!, result.environment)) { + throw new Error("Memmy XeLaTeX runtime is incomplete: engine, review packages or Chinese fonts are unavailable; retry installation."); + } + result.pathEntries.push(...new Set(Object.values(installed.tools).map(file => dirname(file)))); + result.readRoots.push(installed.root); result.executableRoots.push(installed.root); + // A writable TeX cache is granted separately by the command adapter. + result.environment.MEMMY_TEX_CACHE_ROOT = mutableRoot; + return result; +} diff --git a/plugins/office/scripts/package-runtime.mjs b/plugins/office/scripts/package-runtime.mjs index f9dc1345a..c341f463d 100644 --- a/plugins/office/scripts/package-runtime.mjs +++ b/plugins/office/scripts/package-runtime.mjs @@ -10,7 +10,8 @@ import { fileURLToPath } from "node:url"; import JSZip from "jszip"; const hashFile = async file => { const hash = createHash("sha256"); for await (const b of createReadStream(file)) hash.update(b); return hash.digest("hex"); }; -export async function packageRuntime({ root, destination, target, version, tools, setDefault = false }) { +export async function packageRuntime({ root, destination, target, version, tools, setDefault = false, kind = "office" }) { + if (!["office", "texlive"].includes(kind)) throw new Error("Invalid runtime kind"); if (!/^(darwin|linux|win32)-(x64|arm64)$/.test(target) || !/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,80}$/.test(version)) throw new Error("Invalid target/version"); root = await realpath(root); destination = resolve(destination); if (!relative(root, destination).startsWith("..")) throw new Error("Output must be outside the provisioned runtime"); @@ -31,23 +32,23 @@ export async function packageRuntime({ root, destination, target, version, tools } } await walk(root); - if (Object.keys(tools).sort().join(",") !== "pdfinfo,pdftoppm,pdftotext,soffice" || Object.values(tools).some(p => !files[p])) throw new Error("All four tools must be covered by the file manifest"); + if (Object.keys(tools).sort().join(",") !== (kind === "office" ? "pdfinfo,pdftoppm,pdftotext,soffice" : "kpsewhich,xelatex") || Object.values(tools).some(p => !files[p])) throw new Error("All runtime tools must be covered by the file manifest"); await mkdir(destination, { recursive: true }); const temporary = join(destination, `.runtime-${process.pid}-${Date.now()}.tmp`); try { await pipeline(zip.generateNodeStream({ platform: "UNIX", streamFiles: true, compression: "DEFLATE", compressionOptions: { level: 6 } }), createWriteStream(temporary, { flags: "wx" })); const bytes = (await lstat(temporary)).size; if (bytes > 1024 ** 3) throw new Error("Runtime archive exceeds 1 GiB"); - const sha256 = await hashFile(temporary), file = `office-runtime-${target}-${sha256}.zip`; - const manifest = { format: "memmy-office-runtime-v1", target, version, archive: { url: file, sha256, bytes }, tools, files }; - const descriptor = `office-runtime-${target}-${version}.json`, encoded = JSON.stringify(manifest) + "\n"; + const sha256 = await hashFile(temporary), file = `${kind}-runtime-${target}-${sha256}.zip`; + const manifest = { format: `memmy-${kind}-runtime-v1`, target, version, archive: { url: file, sha256, bytes }, tools, files }; + const descriptor = `${kind}-runtime-${target}-${version}.json`, encoded = JSON.stringify(manifest) + "\n"; const previous = await readFile(join(destination, descriptor), "utf8").catch(e => { if (e.code !== "ENOENT") throw e; return null; }); if (previous && previous !== encoded) throw new Error("Runtime releases are immutable; bump the runtime version"); const exists = await lstat(join(destination, file)).catch(() => null); if (exists) { if (await hashFile(join(destination, file)) !== sha256) throw new Error("Published runtime archive is corrupt"); await rm(temporary); } else await rename(temporary, join(destination, file)); if (!previous) await writeFile(join(destination, descriptor), encoded, { flag: "wx" }); - if (setDefault) { const pointer = `${temporary}.json`; await writeFile(pointer, encoded, { flag: "wx" }); await rename(pointer, join(destination, `office-runtime-${target}.json`)); } + if (setDefault) { const pointer = `${temporary}.json`; await writeFile(pointer, encoded, { flag: "wx" }); await rename(pointer, join(destination, `${kind}-runtime-${target}.json`)); } return manifest; } finally { await rm(temporary, { force: true }); } } diff --git a/scripts/texlive/README.md b/scripts/texlive/README.md new file mode 100644 index 000000000..e93b93486 --- /dev/null +++ b/scripts/texlive/README.md @@ -0,0 +1,137 @@ +# Managed review typesetting runtime + +Memmy downloads a versioned, verified portable TeX Live/TinyTeX runtime when a +plugin declaring `runtimeDependencies: ["texlive"]` activates and the Host cannot +find a complete local installation. Installation is user scoped, outside the app +and MPP; it does not use sudo, a system package manager or modify system PATH. + +The Host probes XeLaTeX and the review's required macro packages/fonts. A working +system installation is retained. A managed installation provides XeLaTeX, +kpsewhich, xdvipdfmx, prebuilt XeLaTeX formats, ctex/Fandol Chinese fonts, Latin +Modern fonts and all dependencies of the bundled review templates. Managed +exports use Fandol explicitly instead of platform-specific Chinese fontsets. + +## Provision and package + +Prepare a TeX Live or TinyTeX tree for each supported native platform/architecture. +Install required TeX Live packages in the release build environment, not on end-user +machines. Required packages include xetex, ctex, fandol, fontspec, natbib, microtype, +booktabs, tools, float, hyperref, geometry, xcolor, fancyhdr, caption, colortbl, +environ, trimspaces, psnfss and lm (plus dependencies). Generate the xelatex format +before packaging. Include distribution and package redistribution notices and +required licenses. Match the platform's shared-library dependencies as well as +its executable; a different platform's runtime cannot be validated on macOS. + +Example for a provisioned MacTeX/BasicTeX tree: + +```sh +node scripts/texlive/prepare-runtime.mjs \ + --source /path/to/texlive \ + --destination /tmp/review-texlive \ + --bin bin/universal-darwin +node scripts/texlive/package-runtime.mjs \ + --root /tmp/review-texlive \ + --destination /tmp/review-texlive-release \ + --target darwin-arm64 --version 2026.10.28-1 \ + --tools /tmp/review-texlive/tools.json --set-default +``` + +`--extra` optionally merges an explicitly provisioned texmf tree for missing macro +packages. It must contain distributable TeX packages only. The prepare script +removes symlinks by dereferencing them, omits tex4ht and regenerates filename +indexes. Windows retains the complete native bin directory for DLL dependencies; +Unix release provisioners must supply self-contained native binaries or bundle +all non-system libraries before this step. Smoke-test relocated runtime packages +on each target before publishing. The packager enforces the Host's ZIP limits +and includes SHA-256 digests for every file. Changing an immutable release needs +a new version. + +Publish the ZIP, version descriptor, and `texlive-runtime-{target}.json` pointer +in the literature-review registry's existing root/OSS prefix. The cloud exposes: + +- `GET /api/v1/plugins/literature-review/runtimes/{target}` +- `GET /api/v1/plugins/literature-review/runtimes/{target}/{sha256}` (local storage) + +For OSS the descriptor endpoint replaces the archive name with a signed HTTPS +URL. Raw descriptors must refer only to the matching content-addressed ZIP name. +No plugin-supplied URL grants executable access. + +## Runtime and failure behavior + +Cache defaults to `/plugin-runtimes/texlive/`. Installs are locked, +streamed, bounded, hash checked and atomic. Cached files are reverified on reuse; +fonts and macro resources are covered alongside executables. An installed cache +works offline. Missing or corrupt files trigger a bounded verified reinstall. +Mutable TeX caches live outside the hash-verified runtime and have separate +sandbox write access. Offline users can receive an entire verified cache prepared +by the Host installer for the identical target; cache contents are still verified +before use. There is no end-user offline-import UI yet. + +Host deployment overrides: + +- `MEMMY_TEXLIVE_RUNTIME_CATALOG_URL` — trusted catalog URL (`{target}` placeholder) +- `MEMMY_TEXLIVE_RUNTIME_CACHE` — absolute user-scoped cache root +- `MEMMY_TEXLIVE_RUNTIME_PREFER_MANAGED=1` — use the managed runtime for validation +- `texliveRuntimeInstaller: null` — embedded Host API opt-out for offline deployments/tests + +Failure to acquire/validate the required runtime fails activation with an error +and is retriable by enabling the plugin again. Missing dependencies must not +silently produce an unbranded PDF: the literature plugin now rejects failed +Memmy-layout compilation, retains Markdown/TeX, and deletes stale PDF output. +The other explicitly selected layouts retain their existing labelled fallback. + +## Release requirements + +Publish the backend routes and per-target runtime artifacts before enabling this +Host change in a desktop release. A route without a matching published runtime +will return 404 and activation fails on machines lacking a complete local TeX +installation. Ship the literature plugin change together with the Host change. +Test macOS, Windows and Linux packages on their actual native targets; local +macOS verification is not evidence of Windows/Linux support. No runtime artifact +is added to the MPP or desktop bundle. + +## Native validation + +`.github/workflows/texlive-runtime.yml` builds Windows x64, Linux x64/ARM64, and +macOS ARM64/x64 on native GitHub runners. Linux is a renderer compatibility target; +this does not add a Linux desktop release or certify CLI plugin integration. +Memmy's published desktop installers currently cover macOS ARM64/x64 and Windows x64. + +The provisioner pins official TinyTeX v2026.10 archives and their SHA-256 digests. +Only release builders update/provision TeX packages, synchronize installed font +maps, and generate the XeLaTeX format. The end-user installer only extracts a +verified self-contained runtime. The original build tree is renamed before +validation; the compiler's PATH excludes it. Every target must pass cold install, +offline cache reuse, and two-pass Chinese/English compilation using public brand +fixtures generated by the literature plugin. `verify-pdfs.py` checks Chinese text, +logo images on every page, the orange rules and tinted abstract, and renders all +pages. CI artifacts include the ZIP, descriptors, PDFs, page PNGs and JSON reports. + +Brand fixtures are stored in OSS, not in this repository or the desktop bundle. +`fixtures.lock.json` pins the immutable bundle URL, version, byte size, archive +SHA-256 and the SHA-256 of every extracted file (including source provenance). +CI downloads it with `fetch-fixtures.mjs` before provisioning TeX. Redirects, +changed bytes, unexpected files and unsafe ZIP paths fail validation; there is no +fallback to unverified local fixtures. The public-read object contains only the +previously public generated test documents and brand assets, with no credentials +or user documents. CI needs no OSS credentials. + +To update fixtures, generate the public Chinese/English examples in the literature +plugin, package their TeX/style/logo/figure files and provenance into a ZIP, then +upload a new version/hash-named object under `oss://memos-file/memmy/literature-review/`. +Keep published objects immutable. Review and update the lock's URL, size, archive +hash and per-file hashes together, then rerun all five native jobs. The current +fixture version deliberately preserves plugin 0.5.59's generated examples; it is +independent of the installed plugin's release version. + +To reproduce the installation validation against downloaded native release files: + +```sh +node --test scripts/texlive/fetch-fixtures.test.mjs +node scripts/texlive/fetch-fixtures.mjs FRESH_FIXTURE_DIR +node --import tsx scripts/texlive/smoke-install.ts RELEASE_DIR EMPTY_CACHE_DIR ENV_OUTPUT_JSON FRESH_FIXTURE_DIR +python scripts/texlive/verify-pdfs.py ENV_OUTPUT_JSON.qa +``` + +The PDF validator requires `pymupdf==1.26.5` in a development environment. +Do not place its dependencies in the runtime or MPP. diff --git a/scripts/texlive/fetch-fixtures.mjs b/scripts/texlive/fetch-fixtures.mjs new file mode 100644 index 000000000..3fadcb58c --- /dev/null +++ b/scripts/texlive/fetch-fixtures.mjs @@ -0,0 +1,63 @@ +/** Fetch an immutable, hash-pinned public CI fixture bundle. Never used by Desktop. */ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import JSZip from "jszip"; + +const digest = bytes => createHash("sha256").update(bytes).digest("hex"); +const safeName = name => typeof name === "string" && /^[A-Za-z0-9._/-]+$/u.test(name) + && !name.startsWith("/") && name.split("/").every(part => part && part !== "." && part !== ".."); + +export async function fetchFixtures(destination, options = {}) { + const lock = options.lock ?? JSON.parse(await readFile(new URL("./fixtures.lock.json", import.meta.url), "utf8")); + const fetchFn = options.fetchFn ?? fetch; + const { archive, files } = lock; + const url = new URL(archive.url); + if (lock.format !== "memmy-texlive-brand-fixtures-v1" || url.protocol !== "https:" || url.username || url.password || url.search || url.hash + || !/^[a-f0-9]{64}$/u.test(archive.sha256) || !Number.isSafeInteger(archive.bytes) || archive.bytes < 1 || archive.bytes > 8 * 1024 * 1024 + || !files || !Object.keys(files).length || Object.entries(files).some(([name, hash]) => !safeName(name) || !/^[a-f0-9]{64}$/u.test(hash))) { + throw new Error("Invalid fixture lock"); + } + const response = await fetchFn(url, { redirect: "error", signal: AbortSignal.timeout(60000) }); + if (!response.ok || !response.body) throw new Error(`Fixture download failed: HTTP ${response.status}`); + const chunks = []; let bytes = 0; + for await (const chunk of response.body) { + bytes += chunk.length; + if (bytes > archive.bytes) throw new Error("Fixture archive size mismatch"); + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + if (bytes !== archive.bytes || digest(buffer) !== archive.sha256) throw new Error("Fixture archive integrity check failed"); + const zip = await JSZip.loadAsync(buffer, { checkCRC32: true }); + const entries = Object.values(zip.files); + if (entries.length !== Object.keys(files).length) throw new Error("Fixture archive file set mismatch"); + // Validate the full archive before writing any file. JSZip sanitizes traversal + // names, so check unsafeOriginalName as well as the normalized entry name. + const verified = []; + for (const entry of entries) { + const name = entry.unsafeOriginalName ?? entry.name; + if (!safeName(name) || name !== entry.name || entry.dir || !Object.hasOwn(files, name) + || (Number(entry.unixPermissions) & 0xf000) === 0xa000) throw new Error("Unexpected fixture archive entry"); + const data = await entry.async("nodebuffer"); + if (data.length > 8 * 1024 * 1024) throw new Error("Fixture file exceeds size limit"); + if (digest(data) !== files[name]) throw new Error(`Fixture file integrity check failed: ${name}`); + verified.push([name, data]); + } + const target = resolve(destination); + await mkdir(dirname(target), { recursive: true }); + const staging = await mkdtemp(join(dirname(target), ".texlive-fixtures-")); + try { + for (const [name, data] of verified) { + const path = join(staging, name); await mkdir(dirname(path), { recursive: true }); + await writeFile(path, data, { flag: "wx" }); + } + await rename(staging, target); // Require a fresh destination, never merge old fixtures. + } finally { await rm(staging, { recursive: true, force: true }); } + return { version: lock.version, archiveSha256: archive.sha256, bytes, files: verified.length }; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + if (!process.argv[2]) throw new Error("Required: fresh fixture output directory"); + console.log(JSON.stringify(await fetchFixtures(process.argv[2]))); +} diff --git a/scripts/texlive/fetch-fixtures.test.mjs b/scripts/texlive/fetch-fixtures.test.mjs new file mode 100644 index 000000000..c825fff5b --- /dev/null +++ b/scripts/texlive/fetch-fixtures.test.mjs @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import JSZip from "jszip"; +import { fetchFixtures } from "./fetch-fixtures.mjs"; + +const hash = data => createHash("sha256").update(data).digest("hex"); +async function bundle(contents) { + const zip = new JSZip(); + for (const [name, text] of Object.entries(contents)) zip.file(name, text, { createFolders: false }); + return zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); +} +function pin(buffer, contents) { + return { format: "memmy-texlive-brand-fixtures-v1", version: "test-1", + archive: { url: "https://fixtures.test.invalid/pinned.zip", bytes: buffer.length, sha256: hash(buffer) }, + files: Object.fromEntries(Object.entries(contents).map(([name, text]) => [name, hash(text)])) }; +} +async function workspace(t) { + const root = await mkdtemp(join(tmpdir(), "texlive-fixtures-test-")); + t.after(() => rm(root, { recursive: true, force: true })); return root; +} +test("extracts exactly the hash-verified files and refuses an existing output", async t => { + const root = await workspace(t), contents = { "zh-CN/review.tex": "中文", "en/review.tex": "English" }; + const bytes = await bundle(contents), lock = pin(bytes, contents); + const fetchFn = async (url, options) => { assert.equal(url.href, lock.archive.url); assert.equal(options.redirect, "error"); return new Response(bytes); }; + const result = await fetchFixtures(join(root, "out"), { lock, fetchFn }); + assert.equal(result.files, 2); assert.equal(await readFile(join(root, "out/zh-CN/review.tex"), "utf8"), "中文"); + await assert.rejects(fetchFixtures(join(root, "out"), { lock, fetchFn })); +}); +test("rejects archive tampering before any extraction", async t => { + const root = await workspace(t), contents = { "en/review.tex": "original" }, bytes = await bundle(contents); + const tampered = Buffer.from(bytes); tampered[10] ^= 1; + await assert.rejects(fetchFixtures(join(root, "out"), { lock: pin(bytes, contents), fetchFn: async () => new Response(tampered) }), /integrity/); + assert.deepEqual(await readdir(root), []); +}); +test("rejects oversized downloads, mismatched file hashes and extra archive entries", async t => { + const root = await workspace(t), contents = { "en/review.tex": "original" }, bytes = await bundle(contents); + await assert.rejects(fetchFixtures(join(root, "out"), { lock: pin(bytes, contents), fetchFn: async () => new Response(Buffer.concat([bytes, Buffer.from("extra")])) }), /size mismatch/); + const lock = pin(bytes, contents); lock.files["en/review.tex"] = "0".repeat(64); + await assert.rejects(fetchFixtures(join(root, "out"), { lock, fetchFn: async () => new Response(bytes) }), /file integrity/); + const extra = await bundle({ ...contents, "extra.txt": "unexpected" }); + await assert.rejects(fetchFixtures(join(root, "out"), { lock: pin(extra, contents), fetchFn: async () => new Response(extra) }), /file set mismatch/); + assert.deepEqual(await readdir(root), []); +}); +test("rejects ZIP traversal names even when JSZip normalizes them", async t => { + const root = await workspace(t), bytes = await bundle({ "../escape.txt": "bad" }); + const lock = pin(bytes, { "escape.txt": "bad" }); + await assert.rejects(fetchFixtures(join(root, "out"), { lock, fetchFn: async () => new Response(bytes) }), /Unexpected fixture/); + assert.deepEqual(await readdir(root), []); +}); diff --git a/scripts/texlive/fixtures.lock.json b/scripts/texlive/fixtures.lock.json new file mode 100644 index 000000000..bdeceb447 --- /dev/null +++ b/scripts/texlive/fixtures.lock.json @@ -0,0 +1,25 @@ +{ + "format": "memmy-texlive-brand-fixtures-v1", + "version": "2026.10.28-1", + "source": { + "repository": "Literature-Review-Plugin", + "pluginVersion": "0.5.59", + "generator": "evals/memmy-render.test.ts" + }, + "archive": { + "url": "https://memos-file.oss-cn-shanghai.aliyuncs.com/memmy/literature-review/texlive-brand-fixtures-2026.10.28-1-d8315b9e941cab0a.zip", + "sha256": "d8315b9e941cab0a0a732396ffc66a09a0ec0688dc56b990f2831e760c311a27", + "bytes": 423001 + }, + "files": { + "en/assets/figures/lifecycle-source.png": "5298dfb9dfc868259de78fbe657ec0af9da16f700c5bb8b8391a9414d3facd9c", + "en/memmy-logo.png": "59d069b5105f9666d272a2fbc63822e6f600a0b7923bf4f6e48b8ff05d9cb3b1", + "en/memmy-review.sty": "1fa5f64d6c6ee2fe912cf574f847fb7cc28ecb2b9109f34408272995d6912e4a", + "en/review.tex": "18319a339018f752641116e8ec4311d56321b052d00293067fce051a342002de", + "provenance.json": "1858596626677ab1a1c8c30eb58d0aabcf3f0b7db1cbf6c5425c184b58638acc", + "zh-CN/assets/figures/lifecycle-source.png": "548d48b95a1a2d5f70ac9ed688d229b4c1514cb1053d600df245c1325292510e", + "zh-CN/memmy-logo.png": "59d069b5105f9666d272a2fbc63822e6f600a0b7923bf4f6e48b8ff05d9cb3b1", + "zh-CN/memmy-review.sty": "1fa5f64d6c6ee2fe912cf574f847fb7cc28ecb2b9109f34408272995d6912e4a", + "zh-CN/review.tex": "b2b8e047aa60c2aa8771c581442d44e401f61a14df3aa5422f44aa543bd8cc47" + } +} diff --git a/scripts/texlive/package-runtime.mjs b/scripts/texlive/package-runtime.mjs new file mode 100644 index 000000000..7be8435e8 --- /dev/null +++ b/scripts/texlive/package-runtime.mjs @@ -0,0 +1,19 @@ +/** Package an already provisioned, relocatable TeX Live/TinyTeX tree. */ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { packageRuntime } from "../../plugins/office/scripts/package-runtime.mjs"; +export const packageTexliveRuntime = options => packageRuntime({ ...options, kind: "texlive" }); +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const args = {}; + for (let i = 2; i < process.argv.length; i++) { + const flag = process.argv[i]; + if (flag === "--set-default") args.setDefault = true; + else if (["--root", "--destination", "--target", "--version", "--tools"].includes(flag)) args[flag.slice(2)] = process.argv[++i]; + else throw new Error(`Unknown flag ${flag}`); + } + if (!args.root || !args.destination || !args.target || !args.version || !args.tools) throw new Error("Required: --root --destination --target --version --tools TOOLS.json [--set-default]"); + args.tools = JSON.parse(await readFile(args.tools, "utf8")); + const manifest = await packageTexliveRuntime(args); + console.log(JSON.stringify({ target: manifest.target, archive: manifest.archive, files: Object.keys(manifest.files).length })); +} diff --git a/scripts/texlive/prepare-runtime.mjs b/scripts/texlive/prepare-runtime.mjs new file mode 100644 index 000000000..045bdb0fd --- /dev/null +++ b/scripts/texlive/prepare-runtime.mjs @@ -0,0 +1,49 @@ +/** Build a relocatable review runtime from a provisioned TeX Live/TinyTeX installation. + * Provision packages in CI before this step; no installation scripts run on user machines. + */ +import { cp, mkdir, readdir, writeFile } from "node:fs/promises"; +import { resolve, join } from "node:path"; +import { fileURLToPath } from "node:url"; +export async function prepareTexliveRuntime({ source, destination, bin, extra }) { + source = resolve(source); destination = resolve(destination); + if (destination === source || destination.startsWith(source + "/")) throw new Error("Destination must be outside the source TeX distribution"); + await mkdir(destination, { recursive: false }); + for (const name of ["texmf-dist", "texmf-var", "texmf-config", "LICENSE.CTAN", "LICENSE.TL", "release-texlive.txt"]) { + await cp(join(source, name), join(destination, name), { recursive: true, dereference: true, + filter: file => !/(?:^|[\\/])(?:tex4ht)(?:[\\/]|$)/.test(file) }); + } + if (extra) await cp(resolve(extra), join(destination, "texmf-local"), { recursive: true, dereference: true }); + const windows = bin.split(/[\\/]/).at(-1) === "windows", suffix = windows ? ".exe" : ""; + const tools = {}; + for (const tool of ["xelatex", "kpsewhich", "xdvipdfmx"]) { + const relative = `${bin}/${tool}${suffix}`; + await cp(join(source, relative), join(destination, relative), { dereference: true }); + if (tool !== "xdvipdfmx") tools[tool] = relative; + } + // TeX Live Windows also needs its adjacent DLLs; retain all, rather than guess loader dependencies. + if (windows) await cp(join(source, bin), join(destination, bin), { recursive: true, dereference: true }); + await writeFile(join(destination, "texmf.cnf"), "TEXMFLOCAL = $SELFAUTOPARENT/texmf-local\nTEXMFHOME = $SELFAUTOPARENT/texmf-empty\n"); + // Rebuild search databases after filtering and adding macro packages. + for (const tree of ["texmf-dist", "texmf-var", "texmf-config", ...(extra ? ["texmf-local"] : [])]) { + const lines = ["% ls-R -- filename database for kpathsea; do not change this line."]; + async function scan(directory, relative = ".") { + const entries = (await readdir(directory, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name, "en")); + lines.push("", relative + ":", ...entries.map(entry => entry.name)); + for (const entry of entries) if (entry.isDirectory()) await scan(join(directory, entry.name), relative + "/" + entry.name); + } + await scan(join(destination, tree)); + await writeFile(join(destination, tree, "ls-R"), lines.join("\n") + "\n"); + } + await writeFile(join(destination, "tools.json"), JSON.stringify(tools, null, 2) + "\n"); + return tools; +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const args = {}; + for (let i = 2; i < process.argv.length; i += 2) { + const key = process.argv[i]; + if (!["--source", "--destination", "--bin", "--extra"].includes(key)) throw new Error(`Unknown flag ${key}`); + args[key.slice(2)] = process.argv[i + 1]; + } + if (!args.source || !args.destination || !args.bin) throw new Error("Required: --source --destination --bin [--extra]"); + console.log(JSON.stringify(await prepareTexliveRuntime(args))); +} diff --git a/scripts/texlive/provision-runtime.mjs b/scripts/texlive/provision-runtime.mjs new file mode 100644 index 000000000..d3d0a239b --- /dev/null +++ b/scripts/texlive/provision-runtime.mjs @@ -0,0 +1,45 @@ +/** Release-build inputs only: pinned official TinyTeX archives, verified before extraction. */ +import { createHash } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { access, mkdir, readdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { Readable } from "node:stream"; +import { execFileSync } from "node:child_process"; +const inputs = { + "darwin-arm64": ["darwin", "e40c294efae81026d8cc8d12c09d2dd456d268efe7fe092410478bc1973fd40f", 67189208], + "darwin-x64": ["darwin", "e40c294efae81026d8cc8d12c09d2dd456d268efe7fe092410478bc1973fd40f", 67189208], + "linux-x64": ["linux-x86_64", "4d519d6236ee6798e3ec0d8e2093d1fda01aeb267aa22c2a5586d76bcfce6566", 53589988], + "linux-arm64": ["linux-arm64", "1bc5465c7a7c6f13e13c1b54a52147577eb3f389033af0444bc097698eb6653f", 53793596], + "win32-x64": ["windows", "36c329c644683d981a668419d13c8804696e2816ae4d287ad755788fee231a31", 73701889] +}; +const target = `${process.platform}-${process.arch}`, input = inputs[target]; +if (!input) throw new Error(`No native TinyTeX input is available for ${target}`); +const work = resolve(process.argv[2] ?? ""); +if (!process.argv[2]) throw new Error("Required: fresh build-input directory"); +await mkdir(work, { recursive: false }); +const [platform, expected, size] = input, extension = process.platform === "win32" ? "exe" : "tar.xz"; +const filename = `TinyTeX-1-${platform}-v2026.10.${extension}`; +const url = `https://github.com/rstudio/tinytex-releases/releases/download/v2026.10/${filename}`; +const response = await fetch(url, { signal: AbortSignal.timeout(300000) }); +if (!response.ok || !response.body) throw new Error(`TinyTeX download failed: HTTP ${response.status}`); +const archive = join(work, filename); +await pipeline(Readable.fromWeb(response.body), createWriteStream(archive, { flags: "wx" })); +const hash = createHash("sha256"); let received = 0; +for await (const chunk of createReadStream(archive)) { hash.update(chunk); received += chunk.length; } +if (received !== size || hash.digest("hex") !== expected) throw new Error("Pinned TinyTeX input integrity check failed"); +if (process.platform === "win32") { + execFileSync("C:\\Program Files\\7-Zip\\7z.exe", ["x", archive, `-o${work}`, "-y"], { stdio: "inherit" }); +} else execFileSync("tar", ["-xJf", archive, "-C", work], { stdio: "inherit" }); +const names = await readdir(work); +const rootName = names.find(name => ["TinyTeX", ".TinyTeX"].includes(name)); +if (!rootName) throw new Error("TinyTeX distribution root was not found"); +const source = join(work, rootName), bins = await readdir(join(source, "bin")); +const bin = `bin/${process.platform === "win32" ? "windows" : bins.find(name => name.includes(process.platform === "darwin" ? "darwin" : process.arch === "arm64" ? "aarch64" : "x86_64"))}`; +await access(join(source, bin, process.platform === "win32" ? "tlmgr.bat" : "tlmgr")); +await writeFile(join(work, "provenance.json"), JSON.stringify({ url, sha256: expected, bytes: size, target, source, bin }, null, 2)); +if (process.env.GITHUB_ENV) { + const { appendFile } = await import("node:fs/promises"); + await appendFile(process.env.GITHUB_ENV, `TEXLIVE_BUILD_SOURCE=${source}\nTEXLIVE_BUILD_BIN=${bin}\n`); +} +console.log(`Pinned TinyTeX input verified and extracted for ${target}: ${source}`); diff --git a/scripts/texlive/smoke-install.ts b/scripts/texlive/smoke-install.ts new file mode 100644 index 000000000..8fa150fe6 --- /dev/null +++ b/scripts/texlive/smoke-install.ts @@ -0,0 +1,66 @@ +/** Install an actual native release via the Host's download/verification path. + * Registry/archive responses are streamed from a local release directory, so this + * validation needs no published endpoint. It writes the resolved child env as JSON. + * Usage: tsx scripts/texlive/smoke-install.ts RELEASE_DIR CACHE_DIR OUTPUT_ENV_JSON VERIFIED_FIXTURE_DIR + */ +import { createReadStream } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve, basename } from "node:path"; +import { Readable } from "node:stream"; +import { createTexliveRuntimeInstaller, resolveTexliveRuntime } from "../../App/backend/src/adapters/outbound/plugin-runtime/texlive-installer.js"; +const [release, cache, output, fixtures] = process.argv.slice(2); +if (!release || !cache || !output || !fixtures) throw new Error("Required: RELEASE_DIR CACHE_DIR OUTPUT_ENV_JSON VERIFIED_FIXTURE_DIR"); +const target = `${process.platform}-${process.arch}`; +let downloads = 0; +const installer = createTexliveRuntimeInstaller({ + cacheRoot: resolve(cache), catalogUrl: "https://runtime.test.invalid/{target}.json", + fetchFn: (async input => { + const url = new URL(String(input)); + if (url.pathname.endsWith(".json")) { + const manifest = JSON.parse(await readFile(resolve(release, `texlive-runtime-${target}.json`), "utf8")); + return Response.json(manifest); + } + downloads++; + return new Response(Readable.toWeb(createReadStream(resolve(release, basename(url.pathname)))) as ReadableStream); + }) as typeof fetch, + onProgress(stage) { if (stage !== "downloading") process.stdout.write(`TeX runtime: ${stage}\n`); } +}); +process.env.MEMMY_TEXLIVE_RUNTIME_PREFER_MANAGED = "1"; +const runtime = await resolveTexliveRuntime(process.platform, installer); +const offline = createTexliveRuntimeInstaller({ cacheRoot: resolve(cache), catalogUrl: "https://runtime.test.invalid/{target}.json", + fetchFn: (async () => { throw new Error("offline cache reuse unexpectedly requested network"); }) as typeof fetch }); +await resolveTexliveRuntime(process.platform, offline); +if (downloads !== 1) throw new Error(`Expected one archive download, got ${downloads}`); +await writeFile(resolve(output), JSON.stringify({ ...runtime.environment, + PATH: [...runtime.pathEntries, process.env.PATH].join(process.platform === "win32" ? ";" : ":") }, null, 2)); +console.log(`Native ${target} installation, package/font probe, and offline reuse passed.`); + +// Relocated, managed engines compile the plugin's generated brand fixtures twice. +// Exclude provisioned TeX from PATH, so helper binaries cannot fall back to it. +const { execFile } = await import("node:child_process"); +const { promisify } = await import("node:util"); +const { cp, mkdir } = await import("node:fs/promises"); +const run = promisify(execFile); +const fixtureRoot = resolve(fixtures); +const qa = resolve(output + ".qa"); await mkdir(qa, { recursive: true }); +const systemPath = process.platform === "win32" ? [process.env.SystemRoot + "\\System32", process.env.SystemRoot].join(";") : "/usr/bin:/bin:/usr/sbin:/sbin"; +const cleanEnv = { ...process.env, ...runtime.environment, PATH: [...runtime.pathEntries, systemPath].join(process.platform === "win32" ? ";" : ":") }; +for (const key of ["TEXMF", "TEXMFCNF", "TEXINPUTS", "TEXFONTS", "TEXFORMATS"]) delete cleanEnv[key]; +const engine = resolve(runtime.pathEntries[0]!, process.platform === "win32" ? "xelatex.exe" : "xelatex"); +const report = { target, downloads, offlineReuse: true, languages: [] as Array<{ language: string; bytes: number }> }; +for (const language of ["zh-CN", "en"]) { + const directory = resolve(qa, language); + await cp(resolve(fixtureRoot, language), directory, { recursive: true }); + for (let pass = 0; pass < 2; pass++) { + try { await run(engine, ["-no-shell-escape", "-interaction=nonstopmode", "-halt-on-error", "-file-line-error", "review.tex"], { cwd: directory, env: cleanEnv, timeout: 120000, maxBuffer: 4 * 1024 * 1024, windowsHide: true }); } + catch (error) { console.error((error as { stdout?: string }).stdout); throw error; } + } + const pdf = await readFile(resolve(directory, "review.pdf")); + if (pdf.subarray(0, 5).toString() !== "%PDF-") throw new Error(`Invalid ${language} PDF`); + const log = await readFile(resolve(directory, "review.log"), "utf8"); + if (/Missing character:|LaTeX Warning:.*undefined|LaTeX Font Warning:.*not available/i.test(log)) throw new Error(`${language} PDF has missing text/fonts/citations`); + if (process.env.TEXLIVE_BUILD_SOURCE && log.includes(process.env.TEXLIVE_BUILD_SOURCE.replaceAll("\\", "/"))) throw new Error("Compilation escaped to the provisioned source tree"); + report.languages.push({ language, bytes: pdf.length }); +} +await writeFile(resolve(qa, "native-validation.json"), JSON.stringify(report, null, 2)); +console.log(`Native ${target} Chinese/English branded compilation passed: ${qa}`); diff --git a/scripts/texlive/verify-pdfs.py b/scripts/texlive/verify-pdfs.py new file mode 100644 index 000000000..375c5d2b0 --- /dev/null +++ b/scripts/texlive/verify-pdfs.py @@ -0,0 +1,22 @@ +"""Validate actual compiled brand PDFs and render every page for visual QA.""" +import json, pathlib, sys +import fitz +root = pathlib.Path(sys.argv[1]) +report = [] +for language in ('zh-CN', 'en'): + directory = root / language + document = fitz.open(directory / 'review.pdf') + text = ''.join(page.get_text() for page in document) + assert 'PDFKit' not in str(document.metadata), document.metadata + assert '摘要' in text if language == 'zh-CN' else 'Abstract' in text + assert '\ufffd' not in text, 'Replacement glyphs in PDF' + for index, page in enumerate(document): + assert page.get_images(), f'Memmy logo missing on page {index + 1}' + drawings = page.get_drawings() + assert any(d['color'] and abs(d['color'][0] - 253 / 255) < .02 and abs(d['color'][1] - 137 / 255) < .02 for d in drawings), 'Orange brand rule missing' + if index == 0: + assert any(d['fill'] and all(abs(a-b)<.02 for a,b in zip(d['fill'], (1, 247/255, 240/255))) for d in drawings), 'Abstract brand tint missing' + page.get_pixmap(matrix=fitz.Matrix(1.3, 1.3)).save(directory / f'page-{index+1}.png') + report.append({'language': language, 'pages': len(document), 'brandVerified': True}) +(root / 'pdf-validation.json').write_text(json.dumps(report, indent=2) + '\n') +print(json.dumps(report))