fix(workspace): cap instruction discovery during open

src/server.test.ts

@@ -383,6 +383,37 @@ test("open_workspace keeps lifecycle flags out of model output and preserves com
   assert.ok(Array.isArray(card.agents));
 });
 
+test("open_workspace warns about incomplete instruction discovery on initial and reused opens", async (t) => {
+  const context = await fixture(t);
+  const projects = Array.from({ length: 300 }, (_, index) => `project-${index}`);
+  for (const project of projects) {
+    await mkdir(join(context.project, project));
+    await writeFile(join(context.project, project, "AGENTS.md"), "nested instructions\n");
+  }
+
+  const opened = await callOpen(context.client, context.project, "large-workspace");
+  const first = structuredContent(opened);
+  assert.match(first.instruction as string, /available_agents_files is incomplete/);
+  assert.match(responseCard(opened).instruction as string, /ancestor directories/);
+  assert.match(textContent(opened), /even when they are not listed/);
+  assert.ok((first.agents_files as Array<{ content: string }>).some(
+    (file) => file.content === "project instructions\n",
+  ));
+  const available = first.available_agents_files as Array<{ path: string }>;
+  const omitted = projects.find((project) => !available.some(
+    (file) => file.path === `${project}/AGENTS.md`,
+  ));
+  assert.ok(omitted);
+  const read = await context.client.callTool({
+    name: "read", arguments: { workspace_id: first.workspace_id, path: `${omitted}/AGENTS.md` },
+  });
+  assert.match(textContent(read), /nested instructions/);
+
+  const repeated = structuredContent(await callOpen(context.client, context.project, "large-workspace"));
+  assert.equal(repeated.workspace_id, first.workspace_id);
+  assert.match(repeated.instruction as string, /available_agents_files is incomplete/);
+});
+
 test("open_workspace refreshes provider availability for each catalog", async (t) => {
   let available = false;
   const context = await fixture(t, {
@@ -970,6 +1001,14 @@ function structuredContent(result: Awaited<ReturnType<Client["callTool"]>>): Rec
   return result.structuredContent as Record<string, unknown>;
 }
 
+function textContent(result: Awaited<ReturnType<Client["callTool"]>>): string {
+  assert.ok(Array.isArray(result.content));
+  return (result.content as Array<{ type: string; text?: string }>)
+    .filter((block) => block.type === "text")
+    .map((block) => block.text ?? "")
+    .join("\n");
+}
+
 function responseCard(result: Awaited<ReturnType<Client["callTool"]>>): Record<string, unknown> {
   const metadata = result._meta;
   assert.ok(metadata && typeof metadata === "object");

src/server.ts

@@ -445,6 +445,7 @@ function registerMcpSurface(
         workspace,
         agentsFiles,
         availableAgentsFiles,
+        agentsDiscoveryTruncated,
         workspaceReused,
         includeBootstrapContext,
       } = await workspaces.openWorkspace(
@@ -495,18 +496,24 @@ function registerMcpSurface(
       const visibleAgents = includeBootstrapContext ? cardAgents : [];
       const loadedAgentsFiles = includeBootstrapContext ? cardAgentsFiles : [];
       const availableAgentsFileOutputs = includeBootstrapContext ? cardAvailableAgentsFiles : [];
-      const cardInstruction = config.skillsEnabled
+      const discoveryInstruction = agentsDiscoveryTruncated
+        ? "Nested instruction discovery stopped at its scan limit; available_agents_files is incomplete. Before working in a nested directory, check its ancestor directories up to the workspace root for AGENTS.md, AGENTS.MD, CLAUDE.md or CLAUDE.MD and read applicable instructions, even when they are not listed. Opening the target project as its own workspace also loads its root instructions."
+        : "";
+      const cardInstruction = (config.skillsEnabled
         ? "Use this workspace_id for subsequent work in this project. Keep reusing it while working in this project. Follow loaded agents_files instructions. Before working under a path listed in available_agents_files, read that instruction file. When a task matches an available skill in skills, read its path before proceeding."
-        : "Use this workspace_id for subsequent work in this project. Keep reusing it while working in this project. Follow loaded agents_files instructions. Before working under a path listed in available_agents_files, read that instruction file.";
-      const workspaceInstruction = workspaceReused
+        : "Use this workspace_id for subsequent work in this project. Keep reusing it while working in this project. Follow loaded agents_files instructions. Before working under a path listed in available_agents_files, read that instruction file.")
+        + (discoveryInstruction ? ` ${discoveryInstruction}` : "");
+      const workspaceInstruction = (workspaceReused
         ? [
             `Workspace already open as ${workspace.id}.`,
             "Continue with this workspace_id.",
             "Keep following the project instructions, nested instruction files, skills, agent profiles, and diagnostics already provided for this workspace.",
           ].join("\n\n")
         : workspace.mode === "worktree"
           ? "Use this workspace_id for subsequent work in this isolated worktree. Keep reusing it while working in this worktree. Follow the project instructions, nested instruction files, skills, agent profiles, and diagnostics returned for it."
-          : cardInstruction;
+          : cardInstruction)
+        + (discoveryInstruction && (workspaceReused || workspace.mode === "worktree")
+          ? ` ${discoveryInstruction}` : "");
       const instruction = preloadedSubagentInstructions && includeBootstrapContext
         ? [
             workspaceInstruction,

src/workspace-instruction-discovery.test.ts

@@ -0,0 +1,58 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test, { type TestContext } from "node:test";
+import { walkWorkspaceInstructions } from "./workspace-instruction-discovery.js";
+
+const limits = { maxDirectories: 10, maxEntries: 100, maxDurationMs: 60_000 };
+
+test("instruction discovery completes small trees and skips dependency directories", async (t) => {
+  const root = await fixture(t);
+  await mkdir(join(root, "nested"));
+  await mkdir(join(root, "node_modules", "package"), { recursive: true });
+  await writeFile(join(root, "nested", "AGENTS.md"), "nested instructions");
+  await writeFile(join(root, "node_modules", "package", "AGENTS.md"), "dependency instructions");
+  const files: string[] = [];
+  assert.equal(await walkWorkspaceInstructions(root, (path) => { files.push(path); }, limits), true);
+  assert.deepEqual(files, [join(root, "nested", "AGENTS.md")]);
+});
+
+test("directory budget stops the whole walk, including sibling subtrees", async (t) => {
+  const root = await fixture(t);
+  for (const name of ["a", "b", "c"]) {
+    await mkdir(join(root, name));
+    await writeFile(join(root, name, "AGENTS.md"), "instructions");
+  }
+  const files: string[] = [];
+  assert.equal(await walkWorkspaceInstructions(root, (path) => { files.push(path); }, {
+    ...limits, maxDirectories: 2,
+  }), false);
+  assert.equal(files.length, 1);
+});
+
+test("entry budget also bounds a flat directory with no subdirectories", async (t) => {
+  const root = await fixture(t);
+  for (const name of ["a.txt", "b.txt", "c.txt"]) await writeFile(join(root, name), "");
+  let visited = 0;
+  assert.equal(await walkWorkspaceInstructions(root, () => { visited++; }, {
+    ...limits, maxEntries: 2,
+  }), false);
+  assert.equal(visited, 2);
+});
+
+test("expired discovery time budget does not start another directory", async (t) => {
+  const root = await fixture(t);
+  await writeFile(join(root, "AGENTS.md"), "instructions");
+  let visited = 0;
+  assert.equal(await walkWorkspaceInstructions(root, () => { visited++; }, {
+    ...limits, maxDurationMs: 0,
+  }), false);
+  assert.equal(visited, 0);
+});
+
+async function fixture(t: TestContext): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "devspace-instruction-discovery-"));
+  t.after(() => rm(root, { recursive: true, force: true }));
+  return root;
+}

src/workspace-instruction-discovery.ts

@@ -0,0 +1,55 @@
+import { opendir } from "node:fs/promises";
+import { join } from "node:path";
+import { performance } from "node:perf_hooks";
+
+const SKIPPED_CONTEXT_DIRS = new Set([
+  ".git", ".hg", ".svn", ".devspace", "node_modules", "dist", "build",
+  ".next", ".turbo", ".cache",
+]);
+
+interface DiscoveryLimits {
+  maxDirectories: number;
+  maxEntries: number;
+  maxDurationMs: number;
+}
+
+/** Return false when discovery stops early; close directory handles on exit. */
+export async function walkWorkspaceInstructions(
+  root: string,
+  visit: (path: string, name: string) => Promise<void> | void,
+  limits: DiscoveryLimits = {
+    maxDirectories: 256,
+    maxEntries: 10_000,
+    maxDurationMs: 1_000,
+  },
+): Promise<boolean> {
+  const deadline = performance.now() + limits.maxDurationMs;
+  let directories = 0;
+  let entriesVisited = 0;
+
+  async function walk(directory: string): Promise<boolean> {
+    if (directories >= limits.maxDirectories || performance.now() >= deadline) return false;
+    directories++;
+    let entries;
+    try {
+      entries = await opendir(directory);
+    } catch {
+      // Preserve discovery's existing handling of inaccessible directories.
+      return true;
+    }
+
+    for await (const entry of entries) {
+      if (entriesVisited >= limits.maxEntries || performance.now() >= deadline) return false;
+      entriesVisited++;
+      const path = join(directory, entry.name);
+      if (entry.isDirectory()) {
+        if (!SKIPPED_CONTEXT_DIRS.has(entry.name) && !await walk(path)) return false;
+      } else if (entry.isFile()) {
+        await visit(path, entry.name);
+      }
+    }
+    return true;
+  }
+
+  return walk(root);
+}

src/workspaces.test.ts

@@ -31,6 +31,7 @@ test("a checkout exposes initial and nested instruction context", async (t) => {
     opened.availableAgentsFiles.map((file) => file.path),
     [join(context.root, "nested", "AGENTS.md")],
   );
+  assert.equal(opened.agentsDiscoveryTruncated, false);
   assert.deepEqual(
     opened.workspace.agentProfiles.map((profile) => ({
       name: profile.name,

src/workspaces.ts

@@ -7,10 +7,11 @@ import type {
   WorkspaceSession,
   WorkspaceStore,
 } from "./workspace-store.js";
-import { mkdir, opendir, readFile, realpath, stat } from "node:fs/promises";
+import { mkdir, readFile, realpath, stat } from "node:fs/promises";
 import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import { loadProjectContextFiles } from "@earendil-works/pi-coding-agent";
 import type { ServerConfig } from "./config.js";
+import { walkWorkspaceInstructions } from "./workspace-instruction-discovery.js";
 import {
   createManagedWorktree,
   discardRestoredManagedWorktree,
@@ -70,6 +71,7 @@ export interface WorkspaceContext {
   workspace: Workspace;
   agentsFiles: LoadedAgentsFile[];
   availableAgentsFiles: AvailableAgentsFile[];
+  agentsDiscoveryTruncated: boolean;
   workspaceReused: boolean;
   includeBootstrapContext: boolean;
 }
@@ -246,12 +248,14 @@ export class WorkspaceRegistry {
   private async reusedWorkspaceContext(workspace: Workspace): Promise<WorkspaceContext> {
     workspace.agentProfiles = await loadLocalAgentProfiles(this.config, workspace.root);
     const agentsFiles = await this.loadInitialAgentsFiles(workspace.root);
-    const availableAgentsFiles = await this.findAvailableAgentsFiles(workspace.root, agentsFiles);
+    const { files: availableAgentsFiles, truncated: agentsDiscoveryTruncated } =
+      await this.findAvailableAgentsFiles(workspace.root, agentsFiles);
 
     return {
       workspace,
       agentsFiles,
       availableAgentsFiles,
+      agentsDiscoveryTruncated,
       workspaceReused: true,
       includeBootstrapContext: true,
     };
@@ -510,12 +514,14 @@ export class WorkspaceRegistry {
     });
     this.rememberWorkspace(workspace);
     const agentsFiles = await this.loadInitialAgentsFiles(workspace.root);
-    const availableAgentsFiles = await this.findAvailableAgentsFiles(workspace.root, agentsFiles);
+    const { files: availableAgentsFiles, truncated: agentsDiscoveryTruncated } =
+      await this.findAvailableAgentsFiles(workspace.root, agentsFiles);
 
     return {
       workspace,
       agentsFiles,
       availableAgentsFiles,
+      agentsDiscoveryTruncated,
       workspaceReused: false,
       includeBootstrapContext: true,
     };
@@ -610,7 +616,7 @@ export class WorkspaceRegistry {
   private async findAvailableAgentsFiles(
     root: string,
     loadedFiles: LoadedAgentsFile[],
-  ): Promise<AvailableAgentsFile[]> {
+  ): Promise<{ files: AvailableAgentsFile[]; truncated: boolean }> {
     const loadedPaths = new Set(loadedFiles.map((file) => resolve(file.path)));
     const loadedRealPaths = new Set<string>();
     for (const file of loadedFiles) {
@@ -619,17 +625,19 @@ export class WorkspaceRegistry {
     }
     const discovered: AvailableAgentsFile[] = [];
 
-    await walkWorkspace(root, async (path, entry) => {
-      if (!entry.isFile()) return;
-      if (!CONTEXT_FILE_NAMES.has(entry.name)) return;
+    const complete = await walkWorkspaceInstructions(root, async (path, name) => {
+      if (!CONTEXT_FILE_NAMES.has(name)) return;
       if (loadedPaths.has(path)) return;
       const realPath = await tryRealpath(path);
       if (realPath && loadedRealPaths.has(realPath)) return;
 
       discovered.push({ path });
     });
 
-    return discovered.sort((a, b) => a.path.localeCompare(b.path));
+    return {
+      files: discovered.sort((a, b) => a.path.localeCompare(b.path)),
+      truncated: !complete,
+    };
   }
 }
 
@@ -656,18 +664,6 @@ export async function ensureCheckoutWorkspaceRoot(
 }
 
 const CONTEXT_FILE_NAMES = new Set(["AGENTS.md", "AGENTS.MD", "CLAUDE.md", "CLAUDE.MD"]);
-const SKIPPED_CONTEXT_DIRS = new Set([
-  ".git",
-  ".hg",
-  ".svn",
-  ".devspace",
-  "node_modules",
-  "dist",
-  "build",
-  ".next",
-  ".turbo",
-  ".cache",
-]);
 
 export function formatAgentsPath(path: string, workspaceRoot: string | undefined): string {
   if (!workspaceRoot) return path.split(sep).join("/");
@@ -723,30 +719,6 @@ async function tryRealpath(path: string): Promise<string | undefined> {
   }
 }
 
-async function walkWorkspace(
-  directory: string,
-  visit: (path: string, entry: { name: string; isFile(): boolean; isDirectory(): boolean }) => Promise<void> | void,
-): Promise<void> {
-  let entries;
-  try {
-    entries = await opendir(directory);
-  } catch {
-    return;
-  }
-
-  for await (const entry of entries) {
-    const path = join(directory, entry.name);
-    if (entry.isDirectory()) {
-      if (!SKIPPED_CONTEXT_DIRS.has(entry.name)) {
-        await walkWorkspace(path, visit);
-      }
-      continue;
-    }
-
-    await visit(path, entry);
-  }
-}
-
 function isErrnoException(error: unknown): error is NodeJS.ErrnoException {
   return error instanceof Error && "code" in error;
 }