diff --git a/src/graph/extract.ts b/src/graph/extract.ts index d0f9dff19..8796a97ad 100644 --- a/src/graph/extract.ts +++ b/src/graph/extract.ts @@ -309,6 +309,8 @@ const FUNCTION_VALUE_TYPES = new Set([ ]); const EMPTY_SET: ReadonlySet = new Set(); +const EMPTY_MODULE_MAP: ReadonlyMap = new Map(); +const EMPTY_NAMED_IMPORTS: ReadonlyMap = new Map(); const parser = new Parser(); const GRAMMARS: Record = { @@ -335,6 +337,20 @@ export interface WalkCtx { enclosingClass: string | null; // nearest enclosing class (py/ts `self`/`this`) goReceiverVar: string | null; // Go receiver var, e.g. `w` in `func (w *Worker)` importedSymbols: ReadonlyMap; + /** Python only: local name → the dotted module path it may bind, from + * `import a.b as x` / `from a.b import c [as x]`. A call through such a name + * (`x.f()`) is a module-member call: the module names the target file and the + * attribute names the symbol, so resolve.ts can resolve it exactly instead of + * dropping it for want of a receiver type. Empty for every other language. */ + importedModules: ReadonlyMap; + /** Python only: unconditional module-level `from a.b import c [as x]` bindings, + * local name → imported name and module. Kept apart from importedSymbols so they + * feed call resolution only, not identifier references. */ + pythonNamedImports: ReadonlyMap; + /** Python only: names an import binds in scope, from module-level imports + * (conditional ones included) and the enclosing defs' own imports. A call through + * one that neither import map resolves here is dropped, never matched globally. */ + pythonImportedNames: ReadonlySet; // R6 (Phase 2): which list we're inside while walking an `R6Class(...)` call's // arguments — set only for the direct span of a `public =`/`private =`/ // `active =` `list(...)`'s own entries (see walk()'s special-cased `argument` @@ -388,6 +404,14 @@ export function extractFile(rel: string, source: string, lang: Language): Extrac parser.setLanguage(GRAMMARS[lang] as never); const root = parseSource(source); const bindings = collectBindings(root, lang); + const pythonSymbols = new Map(); + const pythonImportedNames = new Set(); + // Module-level rebinding (an assignment, a def or a conditional re-import) drops + // an import binding for the whole file. + const importedModules = lang === "python" + ? withoutPythonShadowedModules(collectPythonImportedModules(root, pythonSymbols, pythonImportedNames), root) + : EMPTY_MODULE_MAP; + const pythonNamedImports = lang === "python" ? withoutPythonShadowedModules(pythonSymbols, root) : EMPTY_NAMED_IMPORTS; const importedSymbols = collectImportedSymbols(root, lang); const rGenerics = lang === "r" ? collectRGenerics(root) : EMPTY_SET; @@ -422,6 +446,9 @@ export function extractFile(rel: string, source: string, lang: Language): Extrac enclosingClass: null, goReceiverVar: null, importedSymbols, + importedModules, + pythonNamedImports, + pythonImportedNames, rR6Access: null, rGenerics, rSuperClass: null, @@ -642,6 +669,18 @@ function walk(node: Parser.SyntaxNode, ctx: WalkCtx, out: NodeV1[], edges: RawEd desc.kind === "function" || desc.kind === "method" ? withoutShadowedImports(ctx.importedSymbols, node) : ctx.importedSymbols, + importedModules: + ctx.lang === "python" && (desc.kind === "function" || desc.kind === "method") + ? withoutPythonShadowedModules(ctx.importedModules, node) + : ctx.importedModules, + pythonNamedImports: + ctx.lang === "python" && (desc.kind === "function" || desc.kind === "method") + ? withoutPythonShadowedModules(ctx.pythonNamedImports, node) + : ctx.pythonNamedImports, + pythonImportedNames: + ctx.lang === "python" && (desc.kind === "function" || desc.kind === "method") + ? withPythonLocalImportNames(ctx.pythonImportedNames, node) + : ctx.pythonImportedNames, // Reset on every new definition — this is a purely local marker for "we're // still inside THIS class-defining call's own public=/private=/active= // argument chain," not something that should leak into a nested definition @@ -662,6 +701,20 @@ function walk(node: Parser.SyntaxNode, ctx: WalkCtx, out: NodeV1[], edges: RawEd : null : ctx.rSuperClass, }; + if (ctx.lang === "python" && (desc.kind === "function" || desc.kind === "method")) { + // Defaults and annotations run in the enclosing scope, before the parameters bind. + const header = [node.childForFieldName("parameters"), node.childForFieldName("return_type")]; + const inHeader = (c: Parser.SyntaxNode): boolean => header.some((h) => sameSyntaxNode(c, h)); + const headerCtx: WalkCtx = { + ...childCtx, + importedModules: ctx.importedModules, + pythonNamedImports: ctx.pythonNamedImports, + pythonImportedNames: ctx.pythonImportedNames, + }; + walkNamedChildren(node.namedChildren.filter(inHeader), headerCtx, out, edges, minted); + walkNamedChildren(node.namedChildren.filter((c) => !inHeader(c)), childCtx, out, edges, minted); + return; + } walkNamedChildren(node.namedChildren, childCtx, out, edges, minted); return; } @@ -765,8 +818,37 @@ function walk(node: Parser.SyntaxNode, ctx: WalkCtx, out: NodeV1[], edges: RawEd implicitSelf: true, }); } else { - const recvType = resolveRecvType(callee.receiver, ctx); - edges.push(recvType ? { ...callEdge, recvType } : callEdge); + // Python `mod.fn()` where `mod` is an imported module alias: the import + // states the file and the attribute states the symbol, which is the same + // two-halves evidence a named import gives a `references` edge. Carry the + // module as the specifier so resolve.ts can look the name up in that file + // alone — no bare-name guessing, and no receiver type to infer. + const importedName = callee.viaMember ? callee.receiver : callee.name; + const imported = ctx.lang === "python" && importedName + ? ctx.pythonNamedImports.get(importedName) + : undefined; + const moduleSpec = + ctx.lang === "python" && callee.viaMember && callee.receiver + ? ctx.importedModules.get(callee.receiver) + : undefined; + // A name an import binds resolves through that import or not at all: shadowed + // here, or bound by an import neither map keeps, it must not fall through to + // receiver typing or global name lookup. + const shadowed = ctx.lang === "python" && !!importedName && ctx.pythonImportedNames.has(importedName) + && (!(imported || moduleSpec) || pythonCallHasLocalBinding(node, importedName)); + const recvType = moduleSpec ? undefined : resolveRecvType(callee.receiver, ctx); + if (imported && !shadowed) { + edges.push({ + ...callEdge, + name: callee.viaMember ? callee.name : imported.name, + specifier: imported.specifier, + ...(callee.viaMember ? { recvType: imported.name } : {}), + }); + } else if (!shadowed) { + // A shadowed bare import must not reconnect through global name lookup. + if (moduleSpec) edges.push({ ...callEdge, specifier: moduleSpec }); + else edges.push(recvType ? { ...callEdge, recvType } : callEdge); + } } } } else if (ctx.lang === "php" && node.type === "use_declaration") { @@ -848,6 +930,260 @@ function walk(node: Parser.SyntaxNode, ctx: WalkCtx, out: NodeV1[], edges: RawEd for (const child of node.namedChildren) walk(child, ctx, out, edges, minted); } +/** + * Python import bindings, as MODULE paths: local name → the dotted path that + * name may denote. `import a.b as x` and `from a.b import c as x` both bind one + * name to one dotted path (`a.b`, `a.b.c`); a relative import keeps its leading + * dots (`from . import c` → `.c`) for resolve.ts to anchor against the importing + * file's directory. + * + * `from a.b import c` is ambiguous in Python's own grammar — `c` is a submodule + * or an exported symbol, and only the file tree can say which. That is resolve.ts's + * job: the path is offered as a candidate here, and a call through it resolves only + * if the module file actually exists. A plain `import a.b` (no alias) binds the + * TOP package name, so the call site reads `a.b.f()` — a two-hop receiver Python's + * callee reader does not report — and is therefore left out rather than guessed at. + */ +function collectPythonImportedModules( + root: Parser.SyntaxNode, + symbols: Map, + importNames: Set, +): Map { + const out = new Map(); + const bind = (aliasNode: Parser.SyntaxNode, path: string): void => { + const names = aliasNode.namedChildren; + if (aliasNode.type === "aliased_import") { + const target = aliasNode.childForFieldName("name") ?? names[0]; + const alias = aliasNode.childForFieldName("alias") ?? names[1]; + if (target && alias) out.set(alias.text, joinModulePath(path, target.text)); + return; + } + // `from a.b import c` binds `c`; a plain `import a.b` binds the top package, + // which the header explains is deliberately left alone. + if (aliasNode.type === "dotted_name" && path) { + out.set(aliasNode.text, joinModulePath(path, aliasNode.text)); + } + }; + const visit = (node: Parser.SyntaxNode, inDefinition: boolean): void => { + if (node.type === "import_statement" || node.type === "import_from_statement") { + const moduleNode = node.type === "import_from_statement" + ? (node.childForFieldName("module_name") ?? + node.namedChildren.find((c) => c.type === "dotted_name" || c.type === "relative_import")) + : undefined; + if (node.type === "import_from_statement" && !moduleNode) return; + // ponytail: only unconditional module-level imports resolve. Conditional ones + // need per-scope binding tables; until then their names are recorded so a call + // through them is dropped rather than guessed. Imports inside a def are added + // per function (withPythonLocalImportNames). + const moduleLevel = node.parent?.type === "module"; + for (const child of node.namedChildren) { + if (sameSyntaxNode(child, moduleNode)) continue; + if (child.type !== "aliased_import" && child.type !== "dotted_name") continue; + const name = child.childForFieldName("name")?.text ?? child.text; + // `import a.b` binds the top package `a`; every other form binds its alias or name. + const local = child.childForFieldName("alias")?.text ?? (moduleNode ? name : name.split(".")[0]); + if (!inDefinition) importNames.add(local); + if (!moduleLevel) continue; + if (moduleNode) { + bind(child, moduleNode.text); + symbols.set(local, { name, specifier: moduleNode.text }); + } else if (child.type === "aliased_import") { + bind(child, ""); + } + } + return; + } + const definition = node.type === "function_definition" || node.type === "class_definition" || node.type === "lambda"; + for (const child of node.namedChildren) visit(child, inDefinition || definition); + }; + visit(root, false); + return out; +} + +/** Names a def's own imports bind, visible in its body and nested defs. */ +function withPythonLocalImportNames(names: ReadonlySet, definition: Parser.SyntaxNode): ReadonlySet { + const body = definition.childForFieldName("body"); + if (!body) return names; + const local = new Set(); + const visit = (node: Parser.SyntaxNode): void => { + if (node.type === "function_definition" || node.type === "class_definition" || node.type === "lambda") return; + if (node.type === "import_statement" || node.type === "import_from_statement") { + const moduleNode = node.childForFieldName("module_name") ?? + node.namedChildren.find((c) => c.type === "dotted_name" || c.type === "relative_import"); + for (const child of node.namedChildren) { + if (node.type === "import_from_statement" && sameSyntaxNode(child, moduleNode)) continue; + const alias = child.type === "aliased_import" ? child.childForFieldName("alias") : null; + if (alias) local.add(alias.text); + else if (child.type === "dotted_name") local.add(node.type === "import_statement" ? child.text.split(".")[0] : child.text); + } + return; + } + for (const child of node.namedChildren) visit(child); + }; + visit(body); + return local.size === 0 ? names : new Set([...names, ...local]); +} + +/** `a.b` + `c` → `a.b.c`; a relative prefix keeps its dots (`.` + `c` → `.c`). */ +function joinModulePath(prefix: string, tail: string): string { + if (!prefix) return tail; + return prefix.endsWith(".") ? `${prefix}${tail}` : `${prefix}.${tail}`; +} + +/** Binding patterns exclude type annotations, default values and attribute names. */ +function collectPythonBindingNames(node: Parser.SyntaxNode, names: Set): void { + if (node.type === "identifier") names.add(node.text); + else if (node.type === "default_parameter" || node.type === "typed_default_parameter") { + const name = node.childForFieldName("name"); + if (name) collectPythonBindingNames(name, names); + } else if (node.type !== "attribute" && node.type !== "subscript") { + const type = node.childForFieldName("type"); + for (const child of node.namedChildren) { + if (!sameSyntaxNode(child, type)) collectPythonBindingNames(child, names); + } + } +} + +/** + * True when a lambda, comprehension or class body between the call and its + * function binds `name`. Those scopes are not definitions, so the import maps + * the walk carries do not account for them. + */ +function pythonCallHasLocalBinding(call: Parser.SyntaxNode, name: string): boolean { + let below: Parser.SyntaxNode = call; + let nested = false; // inside a lambda or comprehension, which cannot see class-body names + for (let scope = call.parent; scope; below = scope, scope = scope.parent) { + let names: Set | undefined; + if (scope.type === "function_definition") { + // Defaults and annotations run in the enclosing scope; a body call stops here. + if (!pythonDefinitionHeader(scope).some((h) => sameSyntaxNode(below, h))) break; + continue; + } + if (scope.type === "lambda") { + const params = scope.childForFieldName("parameters"); + if (sameSyntaxNode(below, params)) continue; + nested = true; + names = new Set(); + if (params) collectPythonBindingNames(params, names); + const body = scope.childForFieldName("body"); + if (body) collectPythonWalrusTargets(body, names); + } else if (scope.type.endsWith("comprehension") || scope.type === "generator_expression") { + const clauses = scope.namedChildren.filter((c) => c.type === "for_in_clause"); + // The leftmost iterable runs in the enclosing scope. + if (clauses.length === 0 || sameSyntaxNode(below, clauses[0])) continue; + nested = true; + names = new Set(); + for (const clause of clauses) { + const left = clause.childForFieldName("left"); + if (left) collectPythonBindingNames(left, names); + } + } else if (scope.type === "class_definition") { + const body = scope.childForFieldName("body"); + if (nested || !body || !sameSyntaxNode(below, body)) continue; + // ponytail: any binding in the class body suppresses the import there, even + // one after the call; order-aware class scoping would keep the earlier call. + names = new Set(); + collectPythonScopeBindings(body, names); + } + if (names?.has(name)) return true; + } + return false; +} + +/** The parts of a def, lambda or class that run in the enclosing scope. */ +function pythonDefinitionHeader(node: Parser.SyntaxNode): Array { + if (node.type === "class_definition") return [node.childForFieldName("superclasses")]; + return [node.childForFieldName("parameters"), node.childForFieldName("return_type")]; +} + +/** Walrus targets bind in the enclosing lambda, through any comprehension inside it. */ +function collectPythonWalrusTargets(node: Parser.SyntaxNode, names: Set): void { + if (node.type === "named_expression") { + const target = node.childForFieldName("name"); + if (target) names.add(target.text); + } + if (node.type === "lambda" || node.type === "function_definition" || node.type === "class_definition") return; + for (const child of node.namedChildren) collectPythonWalrusTargets(child, names); +} + +/** + * Names a scope's own statements bind: assignment, loop, with and except targets, + * walrus, match captures, del, local imports and nested def or class names. + * Nested functions, classes and lambdas are not entered. + */ +function collectPythonScopeBindings(body: Parser.SyntaxNode, names: Set): void { + const visit = (node: Parser.SyntaxNode): void => { + if (node.type === "function_definition" || node.type === "class_definition" || node.type === "lambda") { + const name = node.childForFieldName("name"); + if (name) names.add(name.text); + // A walrus in a default, annotation or base binds here, where the header runs. + for (const part of pythonDefinitionHeader(node)) if (part) collectPythonWalrusTargets(part, names); + return; + } + if (node.type === "type_alias_statement") { + // `type X = ...` or `type X[T] = ...`: the alias is the first `type` child. + const head = node.namedChildren[0]?.namedChildren[0]; + const alias = head?.type === "generic_type" ? head.namedChildren[0] : head; + if (alias?.type === "identifier") names.add(alias.text); + } + if (node.type === "assignment" || node.type === "augmented_assignment" || node.type === "for_statement") { + const left = node.childForFieldName("left"); + if (left) collectPythonBindingNames(left, names); + } + if (node.type === "named_expression") { + const target = node.childForFieldName("name"); + if (target) collectPythonBindingNames(target, names); + } + if (node.type === "as_pattern") { + // `case ... as x` carries no alias field; its name is the last child. + const last = node.lastNamedChild; + const target = node.childForFieldName("alias") ?? (last?.type === "identifier" ? last : null); + if (target) collectPythonBindingNames(target, names); + } + // A bare name in a match pattern is a capture (a dotted name is a value); `_` binds nothing. + if (node.type === "dotted_name" && node.namedChildCount === 1 && node.text !== "_" + && (node.parent?.type === "case_pattern" || node.parent?.type === "keyword_pattern")) { + names.add(node.text); + } + if (node.type === "splat_pattern") { + const target = node.namedChildren[0]; + if (target?.type === "identifier" && target.text !== "_") names.add(target.text); + } + if (node.type === "delete_statement") { + collectPythonBindingNames(node, names); + } + if (node.parent?.type !== "module" && (node.type === "import_statement" || node.type === "import_from_statement")) { + for (const child of node.namedChildren) { + if (child.type === "aliased_import") { + const alias = child.childForFieldName("alias"); + if (alias) names.add(alias.text); + } else if (child.type === "dotted_name" && !sameSyntaxNode(child, node.childForFieldName("module_name"))) { + names.add(child.text.split(".")[0]); + } + } + } + for (const child of node.namedChildren) visit(child); + }; + visit(body); +} + +/** Remove imports rebound in this scope, retaining bindings used by nested scopes. */ +function withoutPythonShadowedModules( + modules: ReadonlyMap, + definition: Parser.SyntaxNode, +): ReadonlyMap { + if (modules.size === 0) return modules; + const shadowed = new Set(); + const params = definition.childForFieldName("parameters"); + if (params) collectPythonBindingNames(params, shadowed); + const body = definition.type === "module" ? definition : definition.childForFieldName("body"); + if (body) collectPythonScopeBindings(body, shadowed); + if (![...shadowed].some((name) => modules.has(name))) return modules; + const out = new Map(modules); + for (const name of shadowed) out.delete(name); + return out; +} + /** * Named imports whose local binding can be recognized later as a symbol use. * Namespace/default imports are intentionally excluded: they do not tell us @@ -2496,7 +2832,12 @@ function importSpecifier(node: Parser.SyntaxNode, lang: Language): string | null if (lang === "python") { const m = node.childForFieldName("module_name") ?? - node.namedChildren.find((c) => c.type === "dotted_name" || c.type === "relative_import"); + node.namedChildren.find((c) => c.type === "dotted_name" || c.type === "relative_import") ?? + // `import a.b as x` wraps the path in an `aliased_import`, so a child-level + // search finds nothing and the import went unrecorded entirely. + node.namedChildren + .find((c) => c.type === "aliased_import") + ?.childForFieldName("name"); return m?.text ?? null; } if (lang === "go") { diff --git a/src/graph/resolve.ts b/src/graph/resolve.ts index c78a61e6c..6a820d954 100644 --- a/src/graph/resolve.ts +++ b/src/graph/resolve.ts @@ -128,6 +128,11 @@ export function resolveEdges( // node ids. A `use App\Models\User` names a PSR-4 class whose file mirrors the namespace // tail under some (unknown) source root, so the suffix is the portable key. const phpFilesBySuffix = new Map(); + // Python module resolution: a file's importable path-suffix (`app/db/session.py` + // → `app/db/session`, `app/db/__init__.py` → `app/db`) → its file node ids. An + // absolute import names a module by dotted path relative to a sys.path root the + // repo never states, so — as for Java and PHP — the suffix is the portable key. + const pythonFilesByModule = new Map(); const hasGoModules = !!opts.goModules?.length; for (const n of nodes) { if (n.kind === "file") { @@ -150,6 +155,13 @@ export function resolveEdges( const parts = toPosixPath(n.path).split("/"); for (let i = 0; i < parts.length; i++) push(phpFilesBySuffix, parts.slice(i).join("/"), n.id); } + if (n.path.endsWith(".py")) { + // `pkg/__init__.py` IS the module `pkg`, so the marker is dropped before + // indexing — an import names the package, never the init file. + const module = toPosixPath(n.path).replace(/\.py$/, "").replace(/\/__init__$/, ""); + const parts = module.split("/"); + for (let i = 0; i < parts.length; i++) push(pythonFilesByModule, parts.slice(i).join("/"), n.id); + } { const p = toPosixPath(n.path); if (p === "lib.rs" || p === "main.rs") rustCrateRoots.push(""); @@ -216,7 +228,9 @@ export function resolveEdges( ? resolveRustUse(e.specifier, e.file, byId, rustCrateRoots) : e.file.endsWith(".php") ? resolvePhpUse(e.specifier, phpFilesBySuffix) - : resolveImport(e.specifier, e.file, byId); + : e.file.endsWith(".py") + ? resolvePythonImport(e.specifier, e.file, byId, pythonFilesByModule) + : resolveImport(e.specifier, e.file, byId); add(e.source, target, "imports", "extracted"); } else if (e.relation === "extends" || e.relation === "implements") { // `implements` also resolves to a `trait` — PHP models trait composition @@ -268,6 +282,46 @@ export function resolveEdges( if (hit && hit.id !== e.source) add(e.source, hit.id, "references", hit.confidence); } } else if (e.relation === "calls") { + // Python imports identify the target module. A named receiver can be a + // class or a submodule; both resolve within their explicit import path. + if (e.specifier && e.name && e.file.endsWith(".py")) { + const moduleFile = resolvePythonImport(e.specifier, e.file, byId, pythonFilesByModule); + let targetFile = moduleFile; + let candidates: NodeV1[]; + if (e.viaMember && e.recvType) { + const owners = (perFileName.get(moduleFile)?.get(e.recvType) ?? []) + .filter((n) => n.id.replace(/~\d+$/, "") === `${moduleFile}#${e.recvType}`); + if (owners.length === 1 && owners[0].kind === "class") { + candidates = (ownerMethod.get(`${e.recvType}.${e.name}`) ?? []) + .filter((n) => n.path === moduleFile && n.id.replace(/~\d+$/, "") === `${owners[0].id}.${e.name}`); + } else if (owners.length === 0) { + // `from pkg import submodule` shares the syntax of a named import. A + // resolved package anchors the submodule to its own directory; an + // ambiguous one drops the call rather than pick a sys.path root. + const spec = e.specifier.endsWith(".") ? `${e.specifier}${e.recvType}` : `${e.specifier}.${e.recvType}`; + if (byId.has(moduleFile)) { + const dir = moduleFile.endsWith("/__init__.py") ? posix.dirname(moduleFile) : null; + // Python's finder tries the package directory before the module file. + targetFile = (dir && [`${dir}/${e.recvType}/__init__.py`, `${dir}/${e.recvType}.py`].find((c) => byId.has(c))) || spec; + } else { + const parentHits = e.specifier.startsWith(".") ? 0 : (pythonFilesByModule.get(e.specifier.replace(/\./g, "/"))?.length ?? 0); + targetFile = parentHits > 1 ? spec : resolvePythonImport(spec, e.file, byId, pythonFilesByModule); + } + candidates = (perFileName.get(targetFile)?.get(e.name) ?? []) + .filter((n) => n.id.replace(/~\d+$/, "") === `${targetFile}#${e.name}` && (n.kind === "function" || n.kind === "class")); + } else { + candidates = []; + } + } else { + candidates = (perFileName.get(targetFile)?.get(e.name) ?? []) + .filter((n) => n.id.replace(/~\d+$/, "") === `${targetFile}#${e.name}` && (n.kind === "function" || n.kind === "class")); + } + if (candidates.length === 1) { + add(e.source, candidates[0].id, "calls", "extracted"); + } + // Explicit imports must not fall through to an unrelated global name. + continue; + } if (e.viaMember) { if (!e.recvType) continue; const hit = resolveTypedMember(e.recvType, e.name!, e.file, ownerMethod, classParents, classTraits, e.argCount); @@ -487,6 +541,41 @@ function resolveTraitMember( return "ambiguous"; } +/** + * Resolve a Python import's dotted module path to an in-repo file node id; + * otherwise return the raw specifier (stdlib or third-party package). + * + * Two shapes, one answer. A RELATIVE import (`.service`, `..core.config`) states + * its anchor: one leading dot is the importing file's own package, each further + * dot climbs one level, so the path is computed, not searched. An ABSOLUTE import + * (`app.db.session`) names a module relative to a sys.path root the repo never + * writes down — it may be the repo root, a `src/` dir, or a package dir — so the + * dotted tail is matched against each file's importable path SUFFIX, exactly the + * root-agnostic key Java and PHP imports use here. + * + * A suffix shared by two files (`a/util.py` and `b/util.py` for `import util`) is + * left unresolved: picking one would invent an edge the source does not state. + */ +function resolvePythonImport( + spec: string, + file: string, + byId: Map, + filesByModule: Map, +): string { + if (spec.startsWith(".")) { + const dots = /^\.+/.exec(spec)![0].length; + const tail = spec.slice(dots).replace(/\./g, "/"); + // One dot = the importing file's own package, so the first dot costs no climb. + const base = posix.dirname(toPosixPath(file)); + const anchor = posix.normalize(posix.join(base, ...Array(dots - 1).fill(".."))); + const target = tail ? posix.join(anchor, tail) : anchor; + for (const c of [`${target}.py`, `${target}/__init__.py`]) if (byId.has(c)) return c; + return spec; + } + const hits = filesByModule.get(spec.replace(/\./g, "/")); + return hits?.length === 1 ? hits[0] : spec; +} + /** * Resolve a module specifier to a file node id when it points inside the repo; * otherwise return the raw specifier (external package or unresolved path). diff --git a/test/graph-python-module-calls.test.ts b/test/graph-python-module-calls.test.ts new file mode 100644 index 000000000..3185b79ce --- /dev/null +++ b/test/graph-python-module-calls.test.ts @@ -0,0 +1,221 @@ +/** + * Tests for Python calls made through an imported MODULE alias — the shape + * `from app.services import billing` / `billing.charge()`, which a layered + * codebase uses for most of its cross-module traffic (importing the module, not + * each function, is what keeps `billing.charge` readable at the call site). + * + * Before this, such a call reached the resolver as a member call with no + * inferable receiver type and was dropped — correctly, by the drop-rather-than- + * guess rule in resolve.ts, since a bare `charge` says nothing about which file + * it lives in. The missing half was the import binding: Python had no + * module-alias table at all, so the module the alias names was never carried to + * the call site. With it, the call states both halves — the file (the import) + * and the symbol (the attribute) — which is the same evidence a named import + * already gives a `references` edge, and resolves exactly. + * + * The negative tests matter more than the positive ones: a module-qualified call + * must not become a licence to guess. A rebound local name, an unresolvable + * module and a name absent from the module it was looked up in must each leave + * the edge unresolved rather than reach for a same-named symbol elsewhere. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { buildGraph } from "../src/graph/build.js"; +import { readGraph, wiringPath } from "../src/graph/write.js"; +import type { GraphV1 } from "../src/graph/types.js"; + +const BILLING = `def charge(user, amount): + return amount + + +def refund(user, amount): + return -amount +`; + +/** A second `charge`, in a module nobody imports here: a bare-name fallback + * would have two candidates, so any edge to THIS one is a guess. */ +const LEGACY = `def charge(user, amount): + return 0 +`; + +const CONFIG = `TIMEOUT = 30 + + +def load(): + return TIMEOUT +`; + +/** Every supported binding shape, plus the two that must NOT resolve. */ +const API = `from app.services import billing +from app.services import billing as money +from app import config as settings +import app.services.billing as direct +from .local import helper + + +def pay(user): + return billing.charge(user, 10) + + +def pay_aliased(user): + return money.refund(user, 10) + + +def read_timeout(): + return settings.load() + + +def pay_direct(user): + return direct.charge(user, 1) + + +def use_relative(): + return helper.run() + + +def shadowed(billing): + # The parameter rebinds the name: this is a call on the argument, not into + # the module, and must not produce an edge into app/services/billing.py. + return billing.charge(1, 2) + + +def rebound(): + billing = _Stub() + return billing.charge(3, 4) + + +def absent(user): + # The module resolves, but it has no "void" — no edge, and no reaching + # for a same-named symbol in another file. + return billing.void(user) + + +class _Stub: + def charge(self, a, b): + return 0 +`; + +const LOCAL_HELPER = `def run(): + return 1 +`; + +/** An import of a module that is not in the repo — the specifier must stay + * unresolved and the call must not fall back to the in-repo `charge`. */ +const EXTERNAL = `import thirdparty.billing as vendor + + +def pay(user): + return vendor.charge(user, 5) +`; + +function makeFixture(): string { + const dir = mkdtempSync(join(tmpdir(), "graft-py-modcalls-")); + mkdirSync(join(dir, "app", "services"), { recursive: true }); + mkdirSync(join(dir, "app", "local"), { recursive: true }); + writeFileSync(join(dir, "app", "__init__.py"), ""); + writeFileSync(join(dir, "app", "config.py"), CONFIG); + writeFileSync(join(dir, "app", "services", "__init__.py"), ""); + writeFileSync(join(dir, "app", "services", "billing.py"), BILLING); + writeFileSync(join(dir, "app", "legacy.py"), LEGACY); + writeFileSync(join(dir, "app", "local", "__init__.py"), ""); + writeFileSync(join(dir, "app", "api.py"), API); + writeFileSync(join(dir, "app", "local", "helper.py"), LOCAL_HELPER); + writeFileSync(join(dir, "app", "external.py"), EXTERNAL); + return dir; +} + +async function buildFixture(dir: string): Promise { + await buildGraph(dir); // $0, Tier-1 only + const graph = readGraph(wiringPath(join(dir, "graft"))); + assert.ok(graph, "wiring graph should be written"); + return graph!; +} + +const callsOf = (graph: GraphV1, source: string): string[] => + graph.edges.filter((e) => e.relation === "calls" && e.source === source).map((e) => e.target); + +test("Python: a call through an imported module alias resolves to that module's function", async () => { + const graph = await buildFixture(makeFixture()); + + assert.deepEqual( + callsOf(graph, "app/api.py#pay"), + ["app/services/billing.py#charge"], + "billing.charge() should resolve into the imported module", + ); + assert.deepEqual( + callsOf(graph, "app/api.py#pay_aliased"), + ["app/services/billing.py#refund"], + "an `as` alias binds the same module", + ); + assert.deepEqual( + callsOf(graph, "app/api.py#read_timeout"), + ["app/config.py#load"], + "`from app import config as settings` binds a module one level up", + ); + assert.deepEqual( + callsOf(graph, "app/api.py#pay_direct"), + ["app/services/billing.py#charge"], + "`import a.b.c as x` binds the dotted module too", + ); + assert.deepEqual( + callsOf(graph, "app/api.py#use_relative"), + ["app/local/helper.py#run"], + "a relative import anchors on the importing file's own package", + ); +}); + +test("Python: a module-qualified call never guesses past its own module", async () => { + const graph = await buildFixture(makeFixture()); + + assert.deepEqual( + callsOf(graph, "app/api.py#shadowed"), + [], + "a parameter of the same name shadows the module for the whole body", + ); + // The local `billing = _Stub()` keeps its own edges (constructor, and the + // stub's method via the receiver-binding table) — what must not appear is an + // edge into the module the name no longer denotes. + assert.ok( + !callsOf(graph, "app/api.py#rebound").some((t) => t.startsWith("app/services/billing.py")), + "a local assignment rebinds the name away from the module", + ); + assert.deepEqual( + callsOf(graph, "app/api.py#absent"), + [], + "a name the module does not define resolves to nothing, not to a namesake elsewhere", + ); + assert.deepEqual( + callsOf(graph, "app/external.py#pay"), + [], + "an out-of-repo module leaves the call unresolved", + ); +}); + +test("Python: absolute and relative imports resolve to in-repo files", async () => { + const graph = await buildFixture(makeFixture()); + const imports = graph.edges + .filter((e) => e.relation === "imports" && e.source === "app/api.py") + .map((e) => e.target); + + // `from app.services import billing` names the PACKAGE, so the import edge + // lands on its `__init__.py` — the module itself is named by the call edge. + // `import app.services.billing as direct` names the module outright. + assert.ok( + imports.includes("app/services/__init__.py"), + `a dotted import should name an in-repo file, got ${JSON.stringify(imports)}`, + ); + assert.ok( + imports.includes("app/services/billing.py"), + `an aliased dotted import should name the module file, got ${JSON.stringify(imports)}`, + ); + assert.ok(imports.includes("app/local/__init__.py"), "a relative import should name the file"); + assert.ok( + graph.edges.some( + (e) => e.relation === "imports" && e.source === "app/external.py" && e.target === "thirdparty.billing", + ), + "an external package stays a truthful unresolved specifier", + ); +}); diff --git a/test/graph-python-named-imports.test.ts b/test/graph-python-named-imports.test.ts new file mode 100644 index 000000000..741bf936c --- /dev/null +++ b/test/graph-python-named-imports.test.ts @@ -0,0 +1,303 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { extractFile } from "../src/graph/extract.js"; +import { resolveEdges } from "../src/graph/resolve.js"; + +const definitions = `def validate_result(value): + return value + +class Workspace: + @classmethod + def open(cls): + return cls() +`; + +function calls(files: Record, source = "consumer.py#run"): string[] { + const extracted = Object.entries(files).map(([path, text]) => extractFile(path, text, "python")); + return resolveEdges(extracted.flatMap((x) => x.nodes), extracted.flatMap((x) => x.rawEdges)) + .filter((e) => e.source === source && e.relation === "calls") + .map((e) => e.target); +} + +test("Python named imports resolve aliases, class methods and duplicate function names", () => { + const files = { + "validators.py": definitions, + "other.py": definitions, + "first/helpers.py": "def parse_value(value):\n return value\n", + "second/helpers.py": "def parse_value(value):\n return value\n", + "consumer.py": `from validators import validate_result as validate_equity_result +from validators import Workspace as Project +from first.helpers import parse_value + +def run(): + validate_equity_result(1) + Project.open() + return parse_value(1) +`, + }; + assert.deepEqual(calls(files), [ + "validators.py#validate_result", "validators.py#Workspace.open", "first/helpers.py#parse_value", + ]); +}); + +test("Python named imports resolve relative modules and package initialisers", () => { + assert.deepEqual(calls({ + "pkg/validators.py": definitions, + "pkg/tools/__init__.py": "def parse_value(value):\n return value\n", + "pkg/consumer.py": `from .validators import Workspace +from .tools import parse_value +def run(): + Workspace.open() + return parse_value(1) +`, + }, "pkg/consumer.py#run"), ["pkg/validators.py#Workspace.open", "pkg/tools/__init__.py#parse_value"]); +}); + +test("Python named imports do not guess past an external, absent or ambiguous target", () => { + for (const module of ["external", "empty", "shared"]) { + assert.deepEqual(calls({ + "validators.py": definitions, + "empty.py": "def unrelated():\n pass\n", + "a/shared.py": definitions, + "b/shared.py": definitions, + "consumer.py": `from ${module} import validate_result, Workspace\ndef run():\n validate_result(1)\n Workspace.open()\n`, + }), [], module); + } +}); + +test("Python named imports stay shadowed by parameters and local bindings", () => { + const bodies = [ + "def run(validate_result):\n return validate_result(1)\n", + "def run():\n validate_result = callback\n return validate_result(1)\n", + "def run():\n validate_result, other = callbacks\n return validate_result(1)\n", + "def run():\n for validate_result in callbacks:\n validate_result(1)\n", + "def run():\n from external import other as validate_result\n return validate_result(1)\n", + "def run(Workspace):\n return Workspace.open()\n", + "def run():\n with manager() as validate_result:\n return validate_result(1)\n", + "def run():\n try:\n pass\n except Error as validate_result:\n validate_result(1)\n", + "def run():\n (validate_result := callback)\n return validate_result(1)\n", + "def run():\n del validate_result\n return validate_result(1)\n", + ]; + for (const body of bodies) { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": "from validators import validate_result, Workspace\n" + body, + }), [], body); + } +}); + +test("Python lambda and comprehension bindings shadow imported names locally", () => { + for (const expression of [ + "lambda validate_result: validate_result(1)", + "[validate_result(1) for validate_result in callbacks]", + "(validate_result(1) for validate_result in callbacks)", + ]) { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": `from validators import validate_result\ndef run():\n return ${expression}\n`, + }), [], expression); + } +}); + +test("Python nested scope assignments do not shadow an outer imported caller", () => { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": `from validators import validate_result +def run(): + def inner(): + validate_result = callback + return validate_result(1) +`, + }), ["validators.py#validate_result"]); +}); + +test("Python module assignments suppress named import resolution", () => { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": "from validators import validate_result\nvalidate_result = callback\ndef run():\n return validate_result(1)\n", + }), []); +}); + +test("Python conditional imports suppress overwritten module bindings", () => { + for (const statement of [ + "from external import validate_result", + "from external import other as validate_result", + "import external as validate_result", + ]) { + assert.deepEqual(calls({ + "validators.py": definitions, + "external.py": "def validate_result(value):\n return value\ndef other(value):\n return value\n", + "consumer.py": `from validators import validate_result +if True: + ${statement} +def run(): + return validate_result(1) +`, + }), [], statement); + } +}); + +test("Python annotations, default values and attribute assignments do not rebind imports", () => { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": `from validators import validate_result, Workspace +def run(value: Workspace = Workspace): + value.validate_result = callback + return validate_result(1) +`, + }), ["validators.py#validate_result"]); +}); + +test("Python function-local imports do not become named imports in another function", () => { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": "def load():\n from validators import validate_result as validate\ndef run():\n return validate(1)\n", + }), []); +}); + +test("Python named imports do not select the first duplicate definition", () => { + assert.deepEqual(calls({ + "validators.py": definitions + definitions, + "consumer.py": "from validators import validate_result, Workspace\ndef run():\n validate_result(1)\n Workspace.open()\n", + }), []); + assert.deepEqual(calls({ + "validators.py": "class Workspace:\n def open(self):\n pass\n def open(self):\n pass\n", + "consumer.py": "from validators import Workspace\ndef run():\n return Workspace.open()\n", + }), []); +}); + +test("Python module aliases come from unconditional module-level imports only", () => { + const billing = "def charge():\n pass\n"; + for (const consumer of [ + "def load():\n from app.services import billing\n\ndef run():\n billing.charge()\n", + "import app.services.billing as billing\nbilling = Stub()\n\ndef run():\n billing.charge()\n", + "from app.services import billing\nif flag:\n from other import billing\n\ndef run():\n billing.charge()\n", + ]) { + assert.deepEqual(calls({ + "app/services/billing.py": billing, + "other/billing.py": billing, + "consumer.py": consumer, + }), [], consumer); + } +}); + +test("Python match captures, lambda walrus targets and class-body bindings shadow imports", () => { + for (const body of [ + "def run(value):\n match value:\n case v:\n return v(1)\n", + "def run(value):\n match value:\n case str() as v:\n return v(1)\n", + "def run(value):\n match value:\n case [v, *rest]:\n return v(1)\n", + "def run():\n return (lambda: ((v := callback), v(1))[-1])()\n", + ]) { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": "from validators import validate_result as v\n" + body, + }), [], body); + } + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": "from validators import validate_result as v\ndef run(value):\n match value:\n case Color.RED:\n return v(1)\n", + }), ["validators.py#validate_result"]); + const classBody = { + "validators.py": definitions, + "consumer.py": `from validators import Workspace as Project +class C: + Project = LocalWorkspace + Project.open() + + def m(self): + return Project.open() +`, + }; + assert.deepEqual(calls(classBody, "consumer.py#C"), []); + // Class-body names are not visible inside methods. + assert.deepEqual(calls(classBody, "consumer.py#C.m"), ["validators.py#Workspace.open"]); +}); + +test("Python walrus targets in definition headers and type aliases rebind imports", () => { + const files = (consumer: string) => ({ + "validators.py": definitions, + "app/services/billing.py": "def charge():\n pass\n", + "consumer.py": consumer, + }); + for (const consumer of [ + "from validators import validate_result as v\ndef bind(x=(v := callback)):\n pass\ndef run():\n return v(1)\n", + "from app.services import billing\ndef bind(x=(billing := Stub())):\n pass\ndef run():\n billing.charge()\n", + "from validators import validate_result as v\ntype v = int\ndef run():\n return v(1)\n", + "from app.services import billing\ntype billing = object\ndef run():\n billing.charge()\n", + ]) { + assert.deepEqual(calls(files(consumer)), [], consumer); + } + // A method or lambda default runs in the class body, so it sees the class-local name. + const classLocal = "from validators import validate_result as v\nclass C:\n v = callback\n"; + assert.deepEqual(calls(files(classLocal + " def m(self, x=v(1)):\n pass\n"), "consumer.py#C.m"), []); + assert.deepEqual(calls(files(classLocal + " f = lambda x=v(1): x\n"), "consumer.py#C"), []); +}); + +test("Python local imports in one def do not hide a same-named function elsewhere", () => { + assert.deepEqual(calls({ + "consumer.py": `def load(): + from thirdparty import parse_value +def parse_value(value): + return value +def run(): + return parse_value(1) +`, + }), ["consumer.py#parse_value"]); +}); + +test("Python defaults and the leftmost comprehension iterable see the enclosing import", () => { + for (const body of [ + "def run(v=v(1)):\n pass\n", + "def run():\n return lambda v=v(1): v\n", + "def run():\n return [v for v in v(1)]\n", + ]) { + assert.deepEqual(calls({ + "validators.py": definitions, + "consumer.py": "from validators import validate_result as v\n" + body, + }), ["validators.py#validate_result"], body); + } +}); + +test("Python names bound by an unresolved import never fall back to a global match", () => { + for (const consumer of [ + "def run():\n from thirdparty import validate_result\n return validate_result(1)\n", + "if flag:\n from thirdparty import validate_result\n\ndef run():\n return validate_result(1)\n", + "def run():\n from thirdparty import Workspace\n return Workspace.open()\n", + ]) { + assert.deepEqual(calls({ "validators.py": definitions, "consumer.py": consumer }), [], consumer); + } +}); + +test("Python submodule imports anchor to their package and drop an ambiguous one", () => { + const f = "def f():\n pass\n"; + const consumer = "from pkg import submodule\ndef run():\n submodule.f()\n"; + assert.deepEqual(calls({ "pkg/submodule.py": f, "consumer.py": consumer }), ["pkg/submodule.py#f"]); + assert.deepEqual(calls({ + "root2/pkg/__init__.py": "", + "root2/pkg/submodule.py": f, + "other/submodule.py": f, + "consumer.py": consumer, + }), ["root2/pkg/submodule.py#f"]); + // Python's finder takes the package directory before a same-named module file. + assert.deepEqual(calls({ + "pkg/__init__.py": "", + "pkg/submodule.py": f, + "pkg/submodule/__init__.py": f, + "consumer.py": consumer, + }), ["pkg/submodule/__init__.py#f"]); + assert.deepEqual(calls({ + "root1/pkg/__init__.py": "", + "root2/pkg/__init__.py": "", + "root2/pkg/submodule.py": f, + "consumer.py": consumer, + }), []); +}); + +test("Python named imports feed call resolution, not identifier references", () => { + const { rawEdges } = extractFile( + "consumer.py", + "from validators import validate_result\ndef run():\n handler = validate_result\n return handler\n", + "python", + ); + assert.deepEqual(rawEdges.filter((e) => e.relation === "references"), []); +});