fix(build): fall back to WASM grammars when native tree-sitter is unavailable (#119)

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## Unreleased
+
+### Fixed
+
+- **`graft init` / `graft build` no longer crash on linux/arm64** when
+  `tree-sitter@0.21.1` has no native prebuild (`No native build was found for
+  platform=linux arch=arm64`). Native parsers still load when they exist
+  (darwin, linux-x64). If `require("tree-sitter")` fails, depth languages
+  (TypeScript/JavaScript, Python, Go, Java, Kotlin, Swift, PHP, R) fall back to the
+  existing WASM breadth grammars. Fidelity is reduced: no bindings /
+  receiver-type resolution; call edges come from `tags.scm` (symbols only if
+  a query is missing). The build prints
+  `native parser unavailable — falling back to WASM (reduced binding fidelity)`
+  and `graft check` / the statusline do not treat it as an error. (#119)
+
 ## 0.18.0
 
 ### Added

src/cli.ts

@@ -547,6 +547,7 @@ program
       { repo: buildRoot },
     );
     for (const e of g.errors) console.error(`✗ ${e}`);
+    for (const w of g.warnings) console.error(`⚠ ${w}`);
 
     const rel = relative(process.cwd(), g.contextDir) || "graft";
     if (process.env.GRAFT_NO_GITIGNORE) {

src/graph/build.ts

@@ -17,7 +17,7 @@ import { readFileSync } from "node:fs";
 import { basename, dirname, resolve } from "node:path";
 import { walkDir } from "../ingest/fs.js";
 import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";
-import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";
+import { extractFile, languageLabelOf, languageOf, nativeFallbackWarning, wasmFallbackLangs, type RawEdge } from "./extract.js";
 import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
 import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
 import { contentHash } from "../util/id.js";
@@ -121,6 +121,8 @@ export interface GraphBuildResult {
   languages: string[];
   meaning: EnrichStats;
   errors: string[];
+  /** Non-fatal notices (native parser WASM fallback, etc.). Not check-failures. */
+  warnings: string[];
 }
 
 /** Every Go module in the repo: each `go.mod`'s declared `module` path and the repo
@@ -192,9 +194,22 @@ export async function buildGraph(
 
   // Breadth tier: WASM grammars load asynchronously, so warm the ones this repo
   // needs ONCE here (buildGraph is async) before the synchronous parse loop below
-  // can call extractGeneric. Depth-tier (native) grammars need no warmup.
+  // can call extractGeneric. Depth-tier (native) grammars need no warmup — unless
+  // they failed to load, in which case the same WASM path covers them (#119).
+  const fallback = new Set(wasmFallbackLangs());
   await warmGenericGrammars(
-    new Set(files.map((f) => genericLangOf(f.abs)?.name).filter((n): n is string => !!n)),
+    new Set(
+      files.flatMap((f) => {
+        const names: string[] = [];
+        const generic = genericLangOf(f.abs)?.name;
+        if (generic) names.push(generic);
+        const lang = languageOf(f.abs);
+        if (lang && fallback.has(lang)) names.push(lang);
+        const container = containerLangOf(f.abs);
+        if (container && fallback.has(container.inner)) names.push(container.inner);
+        return names;
+      }),
+    ),
   );
   // Container tier (.vue and friends) loads its wrapper grammars the same way,
   // for the same reason: extractContainer runs inside the sync loop below.
@@ -406,5 +421,6 @@ export async function buildGraph(
     languages: [...langs].sort(),
     meaning,
     errors,
+    warnings: [nativeFallbackWarning()].filter((w): w is string => w !== null),
   };
 }

src/graph/check.ts

@@ -20,7 +20,7 @@
 import { resolve } from "node:path";
 import { relPosix } from "../util/paths.js";
 import { contextDirFor } from "../context/node-file.js";
-import { extractFile, languageOf } from "./extract.js";
+import { extractFile, languageOf, wasmFallbackLangs } from "./extract.js";
 import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
 import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
 import { listSourceFiles } from "./build.js";
@@ -87,8 +87,18 @@ export async function checkGraph(
   const fpOnlyDirs = readFingerprint(outDir)?.onlyDirs;
   const onlyDirs = fpOnlyDirs && fpOnlyDirs.length > 0 ? new Set(fpOnlyDirs) : undefined;
   const sourceFiles = listSourceFiles(root, outDir, undefined, onlyDirs);
+  const fallback = new Set(wasmFallbackLangs());
   await warmGenericGrammars(
-    new Set(sourceFiles.map((f) => genericLangOf(f)?.name).filter((n): n is string => !!n)),
+    new Set(
+      sourceFiles.flatMap((f) => {
+        const names: string[] = [];
+        const generic = genericLangOf(f)?.name;
+        if (generic) names.push(generic);
+        const lang = languageOf(f);
+        if (lang && fallback.has(lang)) names.push(lang);
+        return names;
+      }),
+    ),
   );
   // Container-tier grammars need the same warmup as the generic ones, for the same
   // reason: extraction below is synchronous. Missing this is what made `graft

src/graph/extract.ts

@@ -5,21 +5,22 @@
  * definition (file, class, function, method, interface, type, enum, and TS
  * arrow-function consts) plus unresolved edge intents. Edge *targets* are
  * resolved against the whole-repo node index later, in build.ts.
+ *
+ * Native tree-sitter addons load via `require` inside a try (issue #119). When
+ * they are missing — linux/arm64 has no prebuild in tree-sitter@0.21.1 —
+ * `extractFile` delegates to the WASM breadth extractor instead of throwing
+ * at module load. Darwin / linux-x64 keep the hand-written walk unchanged.
  */
-import Parser from "tree-sitter";
-import TypeScript from "tree-sitter-typescript";
-import Python from "tree-sitter-python";
-import Go from "tree-sitter-go";
-import R from "tree-sitter-r";
-import Java from "tree-sitter-java";
-import Kotlin from "tree-sitter-kotlin";
-import Swift from "tree-sitter-swift";
-import PHP from "tree-sitter-php";
+import type Parser from "tree-sitter";
+import { createRequire } from "node:module";
 import { basename } from "node:path";
 import { contentHash } from "../util/id.js";
 import { collectBindings, goReceiverVarOf, resolveRecvType, type FileBindings } from "./bindings.js";
+import { extractGeneric } from "./generic.js";
 import type { Kind, NodeV1, Relation } from "./types.js";
 
+const require = createRequire(import.meta.url);
+
 export type Language = "typescript" | "tsx" | "python" | "go" | "java" | "kotlin" | "swift" | "php" | "r";
 
 /**
@@ -83,6 +84,85 @@ export function languageLabelOf(path: string): string | null {
   return entryFor(path)?.label ?? null;
 }
 
+export const NATIVE_FALLBACK_WARNING =
+  "native parser unavailable — falling back to WASM (reduced binding fidelity)";
+
+const DEPTH_LANGS: Language[] = ["typescript", "tsx", "python", "go", "java", "kotlin", "swift", "php", "r"];
+
+function asParserCtor(mod: unknown): (new () => Parser) | null {
+  if (typeof mod === "function") return mod as new () => Parser;
+  if (mod && typeof mod === "object" && "default" in mod) {
+    const d = (mod as { default: unknown }).default;
+    if (typeof d === "function") return d as new () => Parser;
+  }
+  return null;
+}
+
+export interface NativeLoadResult {
+  Parser: (new () => Parser) | null;
+  grammars: Partial<Record<Language, unknown>>;
+  error: string | null;
+}
+
+/** Load native grammars via `require`. Injectable so tests can simulate a missing
+ * prebuild (`No native build was found for platform=linux arch=arm64`) without
+ * unloading this process's real addons. Native available → same grammars as
+ * the old static imports; a throw or a missing Parser ctor → empty grammars. */
+export function loadNativeGrammars(req: (id: string) => unknown = require): NativeLoadResult {
+  try {
+    const ParserCtor = asParserCtor(req("tree-sitter"));
+    if (!ParserCtor) {
+      return { Parser: null, grammars: {}, error: "tree-sitter did not export a Parser constructor" };
+    }
+    const grammars: Partial<Record<Language, unknown>> = {};
+    const one = (id: string): unknown | null => {
+      try {
+        return req(id);
+      } catch {
+        return null;
+      }
+    };
+    const ts = one("tree-sitter-typescript") as { typescript?: unknown; tsx?: unknown } | null;
+    if (ts?.typescript) grammars.typescript = ts.typescript;
+    if (ts?.tsx) grammars.tsx = ts.tsx;
+    const py = one("tree-sitter-python");
+    if (py) grammars.python = py;
+    const go = one("tree-sitter-go");
+    if (go) grammars.go = go;
+    const r = one("tree-sitter-r");
+    if (r) grammars.r = r;
+    const java = one("tree-sitter-java");
+    if (java) grammars.java = java;
+    const kotlin = one("tree-sitter-kotlin");
+    if (kotlin) grammars.kotlin = kotlin;
+    const swift = one("tree-sitter-swift");
+    if (swift) grammars.swift = swift;
+    const php = one("tree-sitter-php") as { php?: unknown } | null;
+    if (php?.php) grammars.php = php.php;
+    return { Parser: ParserCtor, grammars, error: null };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { Parser: null, grammars: {}, error: message };
+  }
+}
+
+const native = loadNativeGrammars();
+const parser: Parser | null = native.Parser ? new native.Parser() : null;
+const GRAMMARS: Partial<Record<Language, unknown>> = native.grammars;
+
+/** Depth-tier languages whose native grammar did not load — warm these as WASM. */
+export function wasmFallbackLangs(): Language[] {
+  if (!parser) return [...DEPTH_LANGS];
+  return DEPTH_LANGS.filter((l) => GRAMMARS[l] === undefined);
+}
+
+/** One-line banner when the native runtime itself failed to load (issue #119).
+ * Missing *individual* grammars still WASM-fall back, but that is not the
+ * linux/arm64 crash and must not warn on darwin where tree-sitter loaded. */
+export function nativeFallbackWarning(): string | null {
+  return parser ? null : NATIVE_FALLBACK_WARNING;
+}
+
 /**
  * An edge whose target isn't resolved yet. build.ts turns these into EdgeV1 by
  * matching `name`/`specifier` against the repo-wide node index.
@@ -310,19 +390,6 @@ const FUNCTION_VALUE_TYPES = new Set([
 
 const EMPTY_SET: ReadonlySet<string> = new Set();
 
-const parser = new Parser();
-const GRAMMARS: Record<Language, unknown> = {
-  typescript: TypeScript.typescript,
-  tsx: TypeScript.tsx,
-  python: Python,
-  go: Go,
-  r: R,
-  java: Java,
-  kotlin: Kotlin,
-  swift: Swift,
-  php: PHP.php,
-};
-
 export interface WalkCtx {
   rel: string;
   source: string;
@@ -381,10 +448,13 @@ interface DefDescriptor {
  * the source in <32 KB slices. Code-unit indexing matches `String.slice`. */
 const PARSE_CHUNK = 16384;
 function parseSource(source: string): Parser.SyntaxNode {
-  return parser.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode;
+  return parser!.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode;
 }
 
-export function extractFile(rel: string, source: string, lang: Language): ExtractResult {
+export function extractFile(rel: string, source: string, lang: Language, opts?: { wasm?: boolean }): ExtractResult {
+  if (opts?.wasm || !parser || GRAMMARS[lang] === undefined) {
+    return extractGeneric(rel, source, lang);
+  }
   parser.setLanguage(GRAMMARS[lang] as never);
   const root = parseSource(source);
   const bindings = collectBindings(root, lang);

src/graph/generic.ts

@@ -60,6 +60,27 @@ export const GENERIC_LANGS: readonly GenericLang[] = [
   { name: "lua", exts: [".lua"], wasm: "lua" },
 ];
 
+/**
+ * WASM rows for depth-tier languages. Used only when the native grammar failed
+ * to load (`extract.ts` wasmFallbackLangs). `genericLangOf` does NOT consult
+ * this table — depth extensions stay owned by `languageOf` so the native path
+ * is unchanged when addons exist. java is already in GENERIC_LANGS.
+ */
+export const DEPTH_WASM_LANGS: readonly GenericLang[] = [
+  { name: "typescript", exts: [".ts", ".mts", ".cts", ".js", ".mjs", ".cjs"], wasm: "typescript" },
+  { name: "tsx", exts: [".tsx", ".jsx"], wasm: "tsx" },
+  { name: "python", exts: [".py", ".pyi"], wasm: "python" },
+  { name: "go", exts: [".go"], wasm: "go" },
+  { name: "kotlin", exts: [".kt", ".kts"], wasm: "kotlin" },
+  { name: "swift", exts: [".swift"], wasm: "swift" },
+  { name: "php", exts: [".php"], wasm: "php" },
+  { name: "r", exts: [".r"], wasm: "r" },
+];
+
+function wasmRow(name: string): GenericLang | undefined {
+  return GENERIC_LANGS.find((l) => l.name === name) ?? DEPTH_WASM_LANGS.find((l) => l.name === name);
+}
+
 const byExt = new Map<string, GenericLang>();
 for (const l of GENERIC_LANGS) for (const e of l.exts) byExt.set(e, l);
 
@@ -122,7 +143,7 @@ function loadQuery(name: string): string | null {
  * grammars are silently skipped (their files then extract as file-only). */
 export async function warmGenericGrammars(langNames: Iterable<string>): Promise<void> {
   const want = new Set(langNames);
-  const need = [...want].filter((n) => !loaded.has(n) && GENERIC_LANGS.some((l) => l.name === n));
+  const need = [...want].filter((n) => !loaded.has(n) && wasmRow(n));
   if (need.length === 0) return;
   if (!tsMod) {
     tsMod = await import("web-tree-sitter");
@@ -131,7 +152,7 @@ export async function warmGenericGrammars(langNames: Iterable<string>): Promise<
   await initPromise;
   const { Language, Query } = tsMod;
   for (const name of need) {
-    const row = GENERIC_LANGS.find((l) => l.name === name)!;
+    const row = wasmRow(name)!;
     const bytes = requireWasm(row.wasm);
     if (!bytes) continue;
     try {

src/graph/queries/go.scm

@@ -0,0 +1,25 @@
+; Depth-tier WASM fallback (#119).
+
+(function_declaration
+  name: (identifier) @name) @definition.function
+
+(method_declaration
+  name: (field_identifier) @name) @definition.method
+
+(type_spec
+  name: (type_identifier) @name
+  type: (struct_type)) @definition.struct
+
+(type_spec
+  name: (type_identifier) @name
+  type: (interface_type)) @definition.interface
+
+(type_spec
+  name: (type_identifier) @name) @definition.type
+
+(call_expression
+  function: [
+    (identifier) @name
+    (selector_expression
+      field: (field_identifier) @name)
+  ]) @reference.call

src/graph/queries/python.scm

@@ -0,0 +1,14 @@
+; Depth-tier WASM fallback (#119).
+
+(function_definition
+  name: (identifier) @name) @definition.function
+
+(class_definition
+  name: (identifier) @name) @definition.class
+
+(call
+  function: [
+    (identifier) @name
+    (attribute
+      attribute: (identifier) @name)
+  ]) @reference.call

src/graph/queries/r.scm

@@ -0,0 +1,17 @@
+; Depth-tier WASM fallback (#119).
+
+(binary_operator
+  lhs: (identifier) @name
+  operator: "<-"
+  rhs: (function_definition)
+) @definition.function
+
+(binary_operator
+  lhs: (identifier) @name
+  operator: "="
+  rhs: (function_definition)
+) @definition.function
+
+(call
+  function: (identifier) @name
+) @reference.call