feat: QA Bot — automated visual QA & video evidence system

TODO.md

@@ -0,0 +1,90 @@
+# QA Bot — TODO
+
+## Principles
+
+1. **Ship incrementally** — each task produces a runnable artifact; no big-bang integration
+2. **Reuse existing infra** — use existing `lib/slack/`, `lib/video/`, `src/cli/echoBunShell`, prbot patterns
+3. **Convention over configuration** — auto-detect repo type; `.qabot.yaml` optional override
+4. **Evidence-first QA** — every verdict must have video or E2E test proof attached
+5. **Isolation** — each QA run gets its own workspace under `/bot/qa-runs/{run-id}/`
+6. **Fail fast, report always** — even crashes produce a partial report with whatever evidence was collected
+7. **Single responsibility per file** — one concern per module, clear interfaces via `types.ts`
+
+---
+
+## Phase 0 — Foundation
+
+### Core Types & Config
+- [x] `bot/qa/types.ts` — shared types (QATask, QAResult, Verdict, AppBootstrap, etc.)
+- [x] `bot/qa/config.ts` — default config (timeouts, resolution, paths, models)
+
+### Orchestrator
+- [x] `bot/qa/orchestrator.ts` — parse task, clone repo, detect bootstrap, spawn agent, collect results
+
+### App Bootstrap
+- [x] `bot/qa/bootstrap/types.ts` — AppBootstrap interface
+- [x] `bot/qa/bootstrap/index.ts` — auto-detect registry
+- [x] `bot/qa/bootstrap/comfyui-frontend.ts` — Vue+Vite, mock backend
+- [x] `bot/qa/bootstrap/generic-vite.ts` — generic Vite project
+- [x] `bot/qa/bootstrap/generic-nextjs.ts` — generic Next.js project
+
+### Browser & Recording
+- [x] `bot/qa/browser/controller.ts` — launch Playwright, manage context/page, cursor injection
+- [x] `bot/qa/browser/recorder.ts` — start/stop video recording, screenshot capture
+
+### QA Agent
+- [x] `bot/qa/qa-agent.ts` — AI agent loop: research → execute → verdict
+
+### Reporting
+- [x] `bot/qa/report/generator.ts` — produce markdown report from QAResult
+- [x] `bot/qa/report/github-commenter.ts` — post report as GitHub issue/PR comment
+
+### CLI
+- [x] `bot/qa/cli.ts` — `prbot qa reproduce|verify|smoke|demo|run` commands
+- [x] Wire into `bot/cli.ts` as `qa` command group
+
+### Install
+- [x] Add `playwright` dependency
+
+---
+
+## Phase 1 — Video & Delivery (next)
+- [ ] `bot/qa/video/encoder.ts` — ffmpeg post-processing (title cards, compression)
+- [ ] `bot/qa/video/uploader.ts` — upload to GCS with signed URLs
+- [ ] `bot/qa/report/slack-poster.ts` — post report + video to Slack thread
+- [ ] Xvfb integration for headed recording in CI/GCP
+
+## Phase 2 — PR Verification
+- [ ] Before/after flow in orchestrator (checkout base → record → checkout head → record → compare)
+- [ ] `bot/qa/video/compositor.ts` — concatenate before/after with title cards
+- [ ] GitHub Check Run integration
+- [ ] Label management (add/remove `qa:*` labels)
+
+## Phase 3 — Multi-Repo & Triggers
+- [ ] GitHub webhook handler in `gh-service/` for `qa:*` labels
+- [ ] Slack intent detection in `bot/index.ts` for QA mentions
+- [ ] `/qa` GitHub comment command handler
+- [ ] Rate limiting & deduplication
+- [ ] `.qabot.yaml` config file parsing
+
+## Phase 4 — Smoke Tests & Scheduling
+- [ ] Default smoke test suites per repo type
+- [ ] Cron scheduler for nightly runs
+- [ ] `prbot qa list` and `prbot qa artifacts` commands
+
+## Phase 5 — High-Quality Video Production
+- [ ] Title card generation with Canvas API
+- [ ] Text overlay narration
+- [ ] Thumbnail generation
+- [ ] `prbot qa demo` full polish
+
+## Phase 6 — Desktop & Electron
+- [ ] `bot/qa/bootstrap/comfyui-desktop.ts`
+- [ ] Electron-specific Playwright handling
+
+## Phase 7 — Advanced
+- [ ] Flaky test detection
+- [ ] Visual regression (pixel diff)
+- [ ] Performance monitoring
+- [ ] Accessibility audit (axe-core)
+- [ ] QA Dashboard web UI

bot/cli.ts

@@ -1132,6 +1132,172 @@ async function main() {
         .demandCommand(1, "Please specify a debug subcommand")
         .help();
     })
+    .command("qa", "Automated QA testing with video evidence", (y) => {
+      return y
+        .command(
+          "reproduce",
+          "Reproduce a bug from a GitHub issue",
+          (y) =>
+            y
+              .option("issue", {
+                alias: "i",
+                type: "string",
+                describe: "GitHub issue (owner/repo#123)",
+                demandOption: true,
+              })
+              .option("branch", {
+                alias: "b",
+                type: "string",
+                describe: "Branch to test (default: main)",
+              })
+              .option("post-slack", {
+                type: "string",
+                describe: "Slack channel to post results",
+              }),
+          async (args) => {
+            const { handleReproduce } = await import("./qa/cli");
+            await handleReproduce({
+              issue: args.issue as string,
+              branch: args.branch as string | undefined,
+              postSlack: args["post-slack"] as string | undefined,
+            });
+          },
+        )
+        .command(
+          "verify",
+          "Verify a PR with before/after comparison",
+          (y) =>
+            y
+              .option("pr", {
+                alias: "p",
+                type: "string",
+                describe: "GitHub PR (owner/repo#123)",
+                demandOption: true,
+              })
+              .option("base", {
+                type: "string",
+                describe: "Base branch for comparison",
+              })
+              .option("head", {
+                type: "string",
+                describe: "Head branch to verify",
+              })
+              .option("post-slack", {
+                type: "string",
+                describe: "Slack channel to post results",
+              }),
+          async (args) => {
+            const { handleVerify } = await import("./qa/cli");
+            await handleVerify({
+              pr: args.pr as string,
+              base: args.base as string | undefined,
+              head: args.head as string | undefined,
+              postSlack: args["post-slack"] as string | undefined,
+            });
+          },
+        )
+        .command(
+          "smoke",
+          "Run smoke tests on a branch",
+          (y) =>
+            y
+              .option("repo", {
+                alias: "r",
+                type: "string",
+                describe: "Repository (owner/repo)",
+                demandOption: true,
+              })
+              .option("branch", {
+                alias: "b",
+                type: "string",
+                describe: "Branch to test (default: main)",
+              })
+              .option("post-slack", {
+                type: "string",
+                describe: "Slack channel to post results",
+              }),
+          async (args) => {
+            const { handleSmoke } = await import("./qa/cli");
+            await handleSmoke({
+              repo: args.repo as string,
+              branch: args.branch as string | undefined,
+              postSlack: args["post-slack"] as string | undefined,
+            });
+          },
+        )
+        .command(
+          "demo",
+          "Record a demo video of a feature",
+          (y) =>
+            y
+              .option("repo", {
+                alias: "r",
+                type: "string",
+                describe: "Repository (owner/repo)",
+                demandOption: true,
+              })
+              .option("branch", {
+                alias: "b",
+                type: "string",
+                describe: "Branch with the feature (default: main)",
+              })
+              .option("prompt", {
+                type: "string",
+                describe: "What to demo",
+                demandOption: true,
+              })
+              .option("post-slack", {
+                type: "string",
+                describe: "Slack channel to post results",
+              }),
+          async (args) => {
+            const { handleDemo } = await import("./qa/cli");
+            await handleDemo({
+              repo: args.repo as string,
+              branch: args.branch as string | undefined,
+              prompt: args.prompt as string,
+              postSlack: args["post-slack"] as string | undefined,
+            });
+          },
+        )
+        .command(
+          "run",
+          "Free-form QA task",
+          (y) =>
+            y
+              .option("repo", {
+                alias: "r",
+                type: "string",
+                describe: "Repository (owner/repo)",
+                demandOption: true,
+              })
+              .option("branch", {
+                alias: "b",
+                type: "string",
+                describe: "Branch to test (default: main)",
+              })
+              .option("prompt", {
+                type: "string",
+                describe: "QA task description",
+                demandOption: true,
+              })
+              .option("post-slack", {
+                type: "string",
+                describe: "Slack channel to post results",
+              }),
+          async (args) => {
+            const { handleRun } = await import("./qa/cli");
+            await handleRun({
+              repo: args.repo as string,
+              branch: args.branch as string | undefined,
+              prompt: args.prompt as string,
+              postSlack: args["post-slack"] as string | undefined,
+            });
+          },
+        )
+        .demandCommand(1, "Please specify a QA subcommand")
+        .help();
+    })
     .demandCommand(1, "Please specify a command")
     .strict()
     .help()
@@ -1157,6 +1323,10 @@ async function main() {
         "  prbot slack download-file -f F123ABC -o ./downloaded.pdf",
         "  prbot slack file-info -f F123ABC",
         "  prbot notion search -q 'ComfyUI setup' -l 5",
+        "  prbot qa reproduce -i 'Comfy-Org/ComfyUI_frontend#10688'",
+        "  prbot qa verify -p 'Comfy-Org/ComfyUI_frontend#9500'",
+        "  prbot qa smoke -r Comfy-Org/ComfyUI_frontend",
+        "  prbot qa demo -r Comfy-Org/ComfyUI_frontend --prompt 'Demo the template browser'",
       ].join("\n"),
     ).argv;
 

bot/qa/bootstrap/comfyui-frontend.ts

@@ -0,0 +1,65 @@
+import { spawn, type ChildProcess } from "child_process";
+import { existsSync } from "fs";
+import { readFile } from "fs/promises";
+import path from "path";
+import type { AppBootstrap, AppProcess } from "./types";
+
+export const comfyuiFrontendBootstrap: AppBootstrap = {
+  name: "comfyui-frontend",
+  defaultBaseUrl: "http://localhost:5173",
+
+  async detect(repoDir) {
+    try {
+      const pkg = JSON.parse(await readFile(path.join(repoDir, "package.json"), "utf-8"));
+      return (
+        pkg.name === "@comfyorg/comfyui-frontend" ||
+        pkg.name === "comfyui-frontend" ||
+        existsSync(path.join(repoDir, "src/components/node"))
+      );
+    } catch {
+      return false;
+    }
+  },
+
+  async install(repoDir) {
+    const proc = spawn("npm", ["ci"], { cwd: repoDir, stdio: "inherit" });
+    await new Promise<void>((resolve, reject) => {
+      proc.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`npm ci exited ${code}`))));
+    });
+
+    const pwProc = spawn("npx", ["playwright", "install", "chromium"], { cwd: repoDir, stdio: "inherit" });
+    await new Promise<void>((resolve, reject) => {
+      pwProc.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`playwright install exited ${code}`))));
+    });
+  },
+
+  async start(repoDir) {
+    const processes: ChildProcess[] = [];
+
+    const devServer = spawn("npm", ["run", "dev"], {
+      cwd: repoDir,
+      stdio: "pipe",
+      env: { ...process.env, BROWSER: "none" },
+    });
+    processes.push(devServer);
+
+    const cleanup = async () => {
+      for (const p of processes) {
+        p.kill("SIGTERM");
+        await new Promise((r) => setTimeout(r, 1000));
+        if (!p.killed) p.kill("SIGKILL");
+      }
+    };
+
+    return { processes, baseUrl: this.defaultBaseUrl, cleanup };
+  },
+
+  async readyCheck(url) {
+    try {
+      const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  },
+};

bot/qa/bootstrap/generic-nextjs.ts

@@ -0,0 +1,54 @@
+import { spawn, type ChildProcess } from "child_process";
+import { existsSync } from "fs";
+import path from "path";
+import type { AppBootstrap, AppProcess } from "./types";
+
+export const genericNextjsBootstrap: AppBootstrap = {
+  name: "generic-nextjs",
+  defaultBaseUrl: "http://localhost:3000",
+
+  async detect(repoDir) {
+    return (
+      existsSync(path.join(repoDir, "next.config.ts")) ||
+      existsSync(path.join(repoDir, "next.config.js")) ||
+      existsSync(path.join(repoDir, "next.config.mjs"))
+    );
+  },
+
+  async install(repoDir) {
+    const proc = spawn("npm", ["ci"], { cwd: repoDir, stdio: "inherit" });
+    await new Promise<void>((resolve, reject) => {
+      proc.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`npm ci exited ${code}`))));
+    });
+  },
+
+  async start(repoDir) {
+    const processes: ChildProcess[] = [];
+
+    const devServer = spawn("npm", ["run", "dev"], {
+      cwd: repoDir,
+      stdio: "pipe",
+      env: { ...process.env, BROWSER: "none" },
+    });
+    processes.push(devServer);
+
+    const cleanup = async () => {
+      for (const p of processes) {
+        p.kill("SIGTERM");
+        await new Promise((r) => setTimeout(r, 1000));
+        if (!p.killed) p.kill("SIGKILL");
+      }
+    };
+
+    return { processes, baseUrl: this.defaultBaseUrl, cleanup };
+  },
+
+  async readyCheck(url) {
+    try {
+      const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  },
+};