fix: bound table rendering, glob expansion and e2e server reuse

.claude/skills/bmad-distillator/scripts/analyze_sources.py

@@ -10,6 +10,8 @@
 
 Accepts: file paths, folder paths (scans recursively for .md/.txt/.yaml/.yml/.json),
 or glob patterns. Skips node_modules, .git, __pycache__, .venv, _bmad-output.
+Glob patterns do not follow directory symlinks under `**` and stop after
+MAX_GLOB_MATCHES paths.
 
 Output JSON structure:
   status: "ok" | "error"
@@ -27,7 +29,7 @@
 from __future__ import annotations
 
 import argparse
-import glob
+import fnmatch
 import json
 import os
 import re
@@ -46,6 +48,11 @@
 # Approximate chars per token for estimation
 CHARS_PER_TOKEN = 4
 
+# Most paths a single glob pattern may match before the rest are dropped
+MAX_GLOB_MATCHES = 10_000
+
+GLOB_MAGIC = re.compile(r"[*?[]")
+
 # Thresholds
 SINGLE_COMPRESSOR_MAX_TOKENS = 15_000
 SINGLE_DISTILLATE_MAX_TOKENS = 5_000
@@ -79,6 +86,105 @@
 ]
 
 
+def _scandir(directory: Path) -> list[os.DirEntry]:
+    try:
+        with os.scandir(directory) as it:
+            return sorted(it, key=lambda e: e.name)
+    except OSError:
+        return []
+
+
+def _is_dir(path: Path | os.DirEntry, follow_symlinks: bool = True) -> bool:
+    """is_dir that treats unreadable entries and symlink loops as non-directories, as glob does."""
+    try:
+        if isinstance(path, Path):
+            return path.is_dir() if follow_symlinks else not path.is_symlink() and path.is_dir()
+        return path.is_dir(follow_symlinks=follow_symlinks)
+    except OSError:
+        return False
+
+
+def iter_glob(pattern: str, limit: int = MAX_GLOB_MATCHES) -> list[Path]:
+    """Expand a glob pattern like glob.glob(pattern, recursive=True), but iteratively,
+    without following directory symlinks under `**`, and returning at most `limit` paths.
+
+    As with glob, `*` and `?` stay within one path segment, `**` matches zero or more
+    directories, hidden names only match a segment that starts with a dot, and a
+    trailing separator matches directories only.
+    """
+    if not pattern:
+        return []
+    dirs_only = pattern.endswith(("/", os.sep))
+    parts = Path(pattern).parts
+    first_magic = next((i for i, p in enumerate(parts) if GLOB_MAGIC.search(p)), len(parts))
+    base = Path(*parts[:first_magic]) if first_magic else Path(".")
+    segments = parts[first_magic:]
+    if not segments:
+        try:
+            exists = base.exists()
+        except OSError:
+            exists = False
+        return [base] if exists and (not dirs_only or _is_dir(base)) else []
+    if not _is_dir(base):
+        return []
+
+    matches: list[Path] = []
+    seen: set[Path] = set()
+    visited: set[tuple[Path, int]] = set()
+    stack: list[tuple[Path, int]] = [(base, 0)]
+
+    def add(path: Path) -> bool:
+        if path not in seen:
+            seen.add(path)
+            matches.append(path)
+        return len(matches) >= limit
+
+    while stack:
+        current, index = stack.pop()
+        if (current, index) in visited:
+            continue
+        visited.add((current, index))
+        if index == len(segments):
+            if dirs_only and not _is_dir(current):
+                continue
+            if add(current):
+                break
+            continue
+        segment = segments[index]
+        last = index == len(segments) - 1
+        if segment == "**":
+            stack.append((current, index + 1))
+            for entry in reversed(_scandir(current)):
+                if entry.name.startswith("."):
+                    continue
+                if _is_dir(entry, follow_symlinks=False):
+                    stack.append((Path(entry.path), index))
+                elif last and not dirs_only and add(Path(entry.path)):
+                    stack.clear()
+                    break
+            continue
+        if segment in (".", ".."):
+            # scandir never lists these, so follow them directly as glob does.
+            target = current / segment
+            if last or _is_dir(target):
+                stack.append((target, index + 1))
+            continue
+        for entry in reversed(_scandir(current)):
+            if entry.name.startswith(".") and not segment.startswith("."):
+                continue
+            if not fnmatch.fnmatch(entry.name, segment):
+                continue
+            if last or _is_dir(entry):
+                stack.append((Path(entry.path), index + 1))
+
+    if len(matches) >= limit:
+        print(
+            f"warning: {pattern!r} matched {limit:,}+ paths; ignoring the rest",
+            file=sys.stderr,
+        )
+    return matches
+
+
 def resolve_inputs(inputs: list[str]) -> list[Path]:
     """Resolve input arguments to a flat list of file paths."""
     files: list[Path] = []
@@ -95,9 +201,7 @@ def resolve_inputs(inputs: list[str]) -> list[Path]:
                         files.append(fp.resolve())
         else:
             # Try as glob
-            matches = glob.glob(inp, recursive=True)
-            for m in sorted(matches):
-                mp = Path(m)
+            for mp in sorted(iter_glob(inp)):
                 if mp.is_file() and mp.suffix.lower() in INCLUDE_EXTENSIONS:
                     files.append(mp.resolve())
     # Deduplicate while preserving order

.claude/skills/bmad-distillator/scripts/tests/test_analyze_sources.py

@@ -13,6 +13,7 @@
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from analyze_sources import (
+    iter_glob,
     resolve_inputs,
     detect_doc_type,
     suggest_groups,
@@ -94,6 +95,82 @@ def test_nonexistent_path(self):
         assert len(result) == 0
 
 
+class TestIterGlob:
+    @pytest.mark.parametrize("pattern", [
+        "*.md",
+        "**/*.md",
+        "sub*/*.md",
+        "**",
+        "subdir/**/*.md",
+        "*/prd-?2.md",
+        "[pr]*.md",
+        "missing/**/*.md",
+    ])
+    def test_matches_glob_on_plain_trees(self, temp_dir, pattern):
+        import glob
+
+        full = str(Path(temp_dir) / pattern)
+        expected = {Path(p) for p in glob.glob(full, recursive=True)}
+        assert set(iter_glob(full)) == expected
+
+    def test_skips_hidden_names_unless_the_segment_starts_with_a_dot(self, temp_dir):
+        hidden = Path(temp_dir) / ".hidden"
+        hidden.mkdir()
+        (hidden / "secret.md").write_text("x")
+        assert hidden / "secret.md" not in iter_glob(str(Path(temp_dir) / "**" / "*.md"))
+        assert hidden / "secret.md" in iter_glob(str(Path(temp_dir) / ".hidden" / "*.md"))
+
+    def test_does_not_follow_directory_symlinks_under_double_star(self, temp_dir):
+        loop = Path(temp_dir) / "subdir" / "loop"
+        loop.symlink_to(Path(temp_dir), target_is_directory=True)
+        (Path(temp_dir) / "subdir" / "loop2").symlink_to(
+            Path(temp_dir), target_is_directory=True
+        )
+        result = iter_glob(str(Path(temp_dir) / "**" / "*.md"))
+        assert not any("loop" in p.parts for p in result)
+        assert Path(temp_dir) / "subdir" / "prd-v2.md" in result
+
+    def test_stops_at_the_limit(self, temp_dir, capsys):
+        for i in range(5):
+            (Path(temp_dir) / f"extra-{i}.md").write_text("x")
+        result = iter_glob(str(Path(temp_dir) / "*.md"), limit=3)
+        assert len(result) == 3
+        assert "matched 3+ paths" in capsys.readouterr().err
+
+    @pytest.mark.parametrize("pattern", [
+        "*/",
+        "**/",
+        "subdir/../*.md",
+        "*/../*.md",
+    ])
+    def test_matches_glob_for_trailing_separators_and_parent_segments(self, temp_dir, pattern):
+        import glob
+
+        full = str(Path(temp_dir) / pattern)
+        expected = {Path(p) for p in glob.glob(full, recursive=True)}
+        assert set(iter_glob(full)) == expected
+
+    def test_trailing_separator_excludes_files(self, temp_dir):
+        result = resolve_inputs([str(Path(temp_dir)) + "/*/"])
+        assert result == []
+
+    def test_skips_symlink_loops_instead_of_raising(self, temp_dir):
+        a = Path(temp_dir) / "loop-a"
+        b = Path(temp_dir) / "loop-b"
+        a.symlink_to(b)
+        b.symlink_to(a)
+        result = iter_glob(str(Path(temp_dir) / "*" / "*.md"))
+        assert Path(temp_dir) / "subdir" / "prd-v2.md" in result
+
+    def test_empty_pattern_matches_nothing(self):
+        assert iter_glob("") == []
+
+    def test_returns_a_literal_path_only_if_it_exists(self, temp_dir):
+        existing = str(Path(temp_dir) / "random.txt")
+        assert iter_glob(existing) == [Path(existing)]
+        assert iter_glob(str(Path(temp_dir) / "nope.txt")) == []
+
+
 class TestDetectDocType:
     @pytest.mark.parametrize("filename,expected", [
         ("product-brief-foo.md", "product-brief"),

_bmad/core/bmad-distillator/scripts/analyze_sources.py

@@ -10,6 +10,8 @@
 
 Accepts: file paths, folder paths (scans recursively for .md/.txt/.yaml/.yml/.json),
 or glob patterns. Skips node_modules, .git, __pycache__, .venv, _bmad-output.
+Glob patterns do not follow directory symlinks under `**` and stop after
+MAX_GLOB_MATCHES paths.
 
 Output JSON structure:
   status: "ok" | "error"
@@ -27,7 +29,7 @@
 from __future__ import annotations
 
 import argparse
-import glob
+import fnmatch
 import json
 import os
 import re
@@ -46,6 +48,11 @@
 # Approximate chars per token for estimation
 CHARS_PER_TOKEN = 4
 
+# Most paths a single glob pattern may match before the rest are dropped
+MAX_GLOB_MATCHES = 10_000
+
+GLOB_MAGIC = re.compile(r"[*?[]")
+
 # Thresholds
 SINGLE_COMPRESSOR_MAX_TOKENS = 15_000
 SINGLE_DISTILLATE_MAX_TOKENS = 5_000
@@ -79,6 +86,105 @@
 ]
 
 
+def _scandir(directory: Path) -> list[os.DirEntry]:
+    try:
+        with os.scandir(directory) as it:
+            return sorted(it, key=lambda e: e.name)
+    except OSError:
+        return []
+
+
+def _is_dir(path: Path | os.DirEntry, follow_symlinks: bool = True) -> bool:
+    """is_dir that treats unreadable entries and symlink loops as non-directories, as glob does."""
+    try:
+        if isinstance(path, Path):
+            return path.is_dir() if follow_symlinks else not path.is_symlink() and path.is_dir()
+        return path.is_dir(follow_symlinks=follow_symlinks)
+    except OSError:
+        return False
+
+
+def iter_glob(pattern: str, limit: int = MAX_GLOB_MATCHES) -> list[Path]:
+    """Expand a glob pattern like glob.glob(pattern, recursive=True), but iteratively,
+    without following directory symlinks under `**`, and returning at most `limit` paths.
+
+    As with glob, `*` and `?` stay within one path segment, `**` matches zero or more
+    directories, hidden names only match a segment that starts with a dot, and a
+    trailing separator matches directories only.
+    """
+    if not pattern:
+        return []
+    dirs_only = pattern.endswith(("/", os.sep))
+    parts = Path(pattern).parts
+    first_magic = next((i for i, p in enumerate(parts) if GLOB_MAGIC.search(p)), len(parts))
+    base = Path(*parts[:first_magic]) if first_magic else Path(".")
+    segments = parts[first_magic:]
+    if not segments:
+        try:
+            exists = base.exists()
+        except OSError:
+            exists = False
+        return [base] if exists and (not dirs_only or _is_dir(base)) else []
+    if not _is_dir(base):
+        return []
+
+    matches: list[Path] = []
+    seen: set[Path] = set()
+    visited: set[tuple[Path, int]] = set()
+    stack: list[tuple[Path, int]] = [(base, 0)]
+
+    def add(path: Path) -> bool:
+        if path not in seen:
+            seen.add(path)
+            matches.append(path)
+        return len(matches) >= limit
+
+    while stack:
+        current, index = stack.pop()
+        if (current, index) in visited:
+            continue
+        visited.add((current, index))
+        if index == len(segments):
+            if dirs_only and not _is_dir(current):
+                continue
+            if add(current):
+                break
+            continue
+        segment = segments[index]
+        last = index == len(segments) - 1
+        if segment == "**":
+            stack.append((current, index + 1))
+            for entry in reversed(_scandir(current)):
+                if entry.name.startswith("."):
+                    continue
+                if _is_dir(entry, follow_symlinks=False):
+                    stack.append((Path(entry.path), index))
+                elif last and not dirs_only and add(Path(entry.path)):
+                    stack.clear()
+                    break
+            continue
+        if segment in (".", ".."):
+            # scandir never lists these, so follow them directly as glob does.
+            target = current / segment
+            if last or _is_dir(target):
+                stack.append((target, index + 1))
+            continue
+        for entry in reversed(_scandir(current)):
+            if entry.name.startswith(".") and not segment.startswith("."):
+                continue
+            if not fnmatch.fnmatch(entry.name, segment):
+                continue
+            if last or _is_dir(entry):
+                stack.append((Path(entry.path), index + 1))
+
+    if len(matches) >= limit:
+        print(
+            f"warning: {pattern!r} matched {limit:,}+ paths; ignoring the rest",
+            file=sys.stderr,
+        )
+    return matches
+
+
 def resolve_inputs(inputs: list[str]) -> list[Path]:
     """Resolve input arguments to a flat list of file paths."""
     files: list[Path] = []
@@ -95,9 +201,7 @@ def resolve_inputs(inputs: list[str]) -> list[Path]:
                         files.append(fp.resolve())
         else:
             # Try as glob
-            matches = glob.glob(inp, recursive=True)
-            for m in sorted(matches):
-                mp = Path(m)
+            for mp in sorted(iter_glob(inp)):
                 if mp.is_file() and mp.suffix.lower() in INCLUDE_EXTENSIONS:
                     files.append(mp.resolve())
     # Deduplicate while preserving order