Add multi-GPU detection/VRAM summing, GPU offload control, and Hugging Face model pull support

Author: voidstackloop

README.md

@@ -43,6 +43,12 @@ Beyond chat, Modelforge includes an **agentic mode** — the model can read/writ
 
 **Agent mode** — see the [dedicated section](#agent-mode) below.
 
+**Models & hardware**
+- Model recommendations based on your actual hardware — RAM and VRAM are detected and summed **across all GPUs**, not just the first one, so multi-GPU machines get accurate suggestions.
+- **GPU offload control** — set how many model layers Ollama offloads to GPU (`num_gpu`) per chat, per project, or as a global default; leave it blank to let Ollama decide automatically.
+- **Pull models directly from Hugging Face** — paste any GGUF model's Hugging Face URL (or `hf.co/user/repo`) into the model search box alongside the curated catalog and Ollama's own library.
+- Models with reliable tool/function-calling support are flagged with a 🔧 badge, so picking a good Agent mode model doesn't require guesswork.
+
 **Customization & control**
 - English and Turkish UI localization.
 - Configurable Ollama host — point at a remote server instead of localhost.

app/src/ollama-manager.ts

@@ -231,6 +231,7 @@ export async function chat(
                     frequency_penalty: options?.frequencyPenalty,
                     presence_penalty: options?.presencePenalty,
                     num_ctx: options?.contextLength,
+                    num_gpu: options?.gpuLayers,
                 },
             }),
             signal,

app/src/providers/types.ts

@@ -56,6 +56,11 @@ export interface ChatOptions {
     // Ollama-only: how much conversation history the model actually processes
     // (num_ctx). Cloud providers fix this per-model and don't expose it via API.
     contextLength?: number;
+    // Ollama-only: how many model layers to offload to GPU (num_gpu).
+    // undefined = let Ollama auto-decide, 0 = force CPU-only, a positive
+    // number = offload that many layers (useful for tuning multi-GPU setups
+    // or freeing VRAM for something else running alongside Ollama).
+    gpuLayers?: number;
 }
 
 export type ChatFn = (

app/src/settings-store.ts

@@ -17,6 +17,8 @@ export interface AppSettings {
     frequencyPenalty: number;
     presencePenalty: number;
     contextLength: number;
+    // undefined = auto (let Ollama decide how many layers to offload to GPU).
+    gpuLayers?: number;
     systemPrompt: string;
     promptPresets: PromptPreset[];
     theme: "light" | "dark" | "system";

app/src/system-specs.test.ts

@@ -0,0 +1,67 @@
+import { describe, it, expect } from "vitest";
+import { recommendModels, type SystemSpecs } from "./system-specs";
+
+function baseSpecs(overrides: Partial<SystemSpecs> = {}): SystemSpecs {
+    return {
+        totalRAMGB: 16,
+        freeRAMGB: 8,
+        cpuModel: "Test CPU",
+        cpuCores: 8,
+        platform: "linux",
+        arch: "x64",
+        gpu: null,
+        gpus: [],
+        totalVramGB: null,
+        ...overrides,
+    };
+}
+
+describe("recommendModels", () => {
+    it("falls back to RAM-based sizing when there's no GPU", () => {
+        const result = recommendModels(baseSpecs({ totalRAMGB: 16 }));
+        expect(result.usableVRAMGB).toBe(0);
+        expect(result.usableRAMGB).toBeCloseTo(11.2);
+    });
+
+    it("sums VRAM across multiple GPUs instead of using only the first one", () => {
+        const specs = baseSpecs({
+            gpu: { name: "GPU 0", vramGB: 24, vendor: "nvidia" },
+            gpus: [
+                { name: "GPU 0", vramGB: 24, vendor: "nvidia" },
+                { name: "GPU 1", vramGB: 24, vendor: "nvidia" },
+            ],
+            totalVramGB: 48,
+        });
+        const result = recommendModels(specs);
+        // 48GB total * 0.9 headroom factor
+        expect(result.usableVRAMGB).toBeCloseTo(43.2);
+    });
+
+    it("uses whichever of RAM or VRAM gives more usable headroom", () => {
+        const specs = baseSpecs({
+            totalRAMGB: 8,
+            gpu: { name: "Big GPU", vramGB: 80, vendor: "nvidia" },
+            gpus: [{ name: "Big GPU", vramGB: 80, vendor: "nvidia" }],
+            totalVramGB: 80,
+        });
+        const result = recommendModels(specs);
+        expect(result.effectiveGB).toBeCloseTo(result.usableVRAMGB);
+        expect(result.usableVRAMGB).toBeGreaterThan(result.usableRAMGB);
+    });
+
+    it("marks a model as runsOnGpu only when it fits within usable VRAM", () => {
+        const specs = baseSpecs({
+            totalRAMGB: 80,
+            gpu: { name: "Small GPU", vramGB: 4, vendor: "nvidia" },
+            gpus: [{ name: "Small GPU", vramGB: 4, vendor: "nvidia" }],
+            totalVramGB: 4,
+        });
+        const result = recommendModels(specs);
+        const tiny = result.models.find((m) => m.name === "llama3.2:1b")!;
+        const large = result.models.find((m) => m.name === "llama3.1:70b")!;
+        expect(tiny.runsOnGpu).toBe(true);
+        expect(large.runsOnGpu).toBe(false);
+        // Still fits overall thanks to the large RAM pool, just not on the GPU.
+        expect(large.fits).toBe(true);
+    });
+});

app/src/system-specs.ts

@@ -14,7 +14,12 @@ export interface SystemSpecs {
     cpuCores: number;
     platform: NodeJS.Platform;
     arch: string;
+    // Kept for backward compatibility with anything expecting a single GPU —
+    // this is just gpus[0]. Prefer `gpus` and `totalVramGB` for anything
+    // multi-GPU-aware.
     gpu: GpuInfo | null;
+    gpus: GpuInfo[];
+    totalVramGB: number | null;
 }
 
 export interface ModelCatalogEntry {
@@ -72,17 +77,25 @@ function execFileP(cmd: string, args: string[]): Promise<string | null> {
     });
 }
 
-async function detectGpu(): Promise<GpuInfo | null> {
-    // NVIDIA tooling works the same way on Windows and Linux when drivers are installed.
+// Returns every GPU nvidia-smi/PowerShell/system_profiler reports, not just
+// the first one — a single-GPU read silently undercounts available VRAM
+// (and therefore which models "fit") on any multi-GPU machine.
+async function detectGpus(): Promise<GpuInfo[]> {
+    // NVIDIA tooling works the same way on Windows and Linux when drivers are
+    // installed, and reports one CSV line per GPU when there's more than one.
     const nvidiaOut = await execFileP("nvidia-smi", [
         "--query-gpu=name,memory.total",
         "--format=csv,noheader,nounits",
     ]);
     if (nvidiaOut) {
-        const [name, memMiB] = nvidiaOut.split(",").map((s) => s.trim());
-        if (name && memMiB && !Number.isNaN(Number(memMiB))) {
-            return { name, vramGB: +(Number(memMiB) / 1024).toFixed(1), vendor: "nvidia" };
+        const gpus: GpuInfo[] = [];
+        for (const line of nvidiaOut.trim().split("\n")) {
+            const [name, memMiB] = line.split(",").map((s) => s.trim());
+            if (name && memMiB && !Number.isNaN(Number(memMiB))) {
+                gpus.push({ name, vramGB: +(Number(memMiB) / 1024).toFixed(1), vendor: "nvidia" });
+            }
         }
+        if (gpus.length > 0) return gpus;
     }
 
     if (os.platform() === "win32") {
@@ -94,17 +107,20 @@ async function detectGpu(): Promise<GpuInfo | null> {
         if (out) {
             try {
                 const parsed = JSON.parse(out);
-                const gpu = Array.isArray(parsed) ? parsed[0] : parsed;
-                // AdapterRAM is a known-buggy 32-bit field on Windows for GPUs with >4GB VRAM,
-                // so only trust it when it lands in a plausible range.
-                const ramGB = gpu?.AdapterRAM ? gpu.AdapterRAM / 1e9 : 0;
-                if (gpu?.Name) {
-                    return {
+                const list = Array.isArray(parsed) ? parsed : [parsed];
+                const gpus: GpuInfo[] = [];
+                for (const gpu of list) {
+                    if (!gpu?.Name) continue;
+                    // AdapterRAM is a known-buggy 32-bit field on Windows for GPUs with >4GB VRAM,
+                    // so only trust it when it lands in a plausible range.
+                    const ramGB = gpu.AdapterRAM ? gpu.AdapterRAM / 1e9 : 0;
+                    gpus.push({
                         name: gpu.Name,
                         vramGB: ramGB > 0 && ramGB < 64 ? +ramGB.toFixed(1) : null,
                         vendor: "unknown",
-                    };
+                    });
                 }
+                if (gpus.length > 0) return gpus;
             } catch {
                 // ignore malformed output
             }
@@ -113,34 +129,43 @@ async function detectGpu(): Promise<GpuInfo | null> {
 
     if (os.platform() === "darwin") {
         const out = await execFileP("system_profiler", ["SPDisplaysDataType"]);
-        const nameMatch = out?.match(/Chipset Model:\s*(.+)/);
-        if (nameMatch) {
-            // Apple Silicon GPUs share unified memory with the CPU rather than dedicated VRAM.
-            return { name: nameMatch[1].trim(), vramGB: null, vendor: "apple" };
+        if (out) {
+            const gpus: GpuInfo[] = [];
+            // Apple Silicon GPUs share unified memory with the CPU rather than
+            // dedicated VRAM, so there's no per-GPU size to report here.
+            for (const match of out.matchAll(/Chipset Model:\s*(.+)/g)) {
+                gpus.push({ name: match[1].trim(), vramGB: null, vendor: "apple" });
+            }
+            if (gpus.length > 0) return gpus;
         }
     }
 
-    return null;
+    return [];
 }
 
 export async function getSpecs(): Promise<SystemSpecs> {
     const cpus = os.cpus() || [];
-    const gpu = await detectGpu();
+    const gpus = await detectGpus();
+    const knownVram = gpus.map((g) => g.vramGB).filter((v): v is number => v !== null);
+    const totalVramGB = knownVram.length > 0 ? +knownVram.reduce((a, b) => a + b, 0).toFixed(1) : null;
+
     return {
         totalRAMGB: +(os.totalmem() / 1e9).toFixed(1),
         freeRAMGB: +(os.freemem() / 1e9).toFixed(1),
         cpuModel: cpus[0] ? cpus[0].model : "Unknown CPU",
         cpuCores: cpus.length,
         platform: os.platform(),
         arch: os.arch(),
-        gpu,
+        gpu: gpus[0] ?? null,
+        gpus,
+        totalVramGB,
     };
 }
 
 export function recommendModels(specs: SystemSpecs): ModelRecommendations {
     // Leave headroom for the OS and the app itself on both RAM and VRAM.
     const usableRAMGB = +(specs.totalRAMGB * 0.7).toFixed(1);
-    const usableVRAMGB = specs.gpu?.vramGB ? +(specs.gpu.vramGB * 0.9).toFixed(1) : 0;
+    const usableVRAMGB = specs.totalVramGB ? +(specs.totalVramGB * 0.9).toFixed(1) : 0;
     const effectiveGB = Math.max(usableRAMGB, usableVRAMGB);
 
     const fitting = MODEL_CATALOG.filter((m) => m.minRAMGB <= effectiveGB);

frontend/src/components/layout.tsx

@@ -271,6 +271,22 @@ function ProjectGroup({
                                         className="h-8 text-xs"
                                     />
                                 </div>
+                                <div className="flex flex-col gap-1">
+                                    <label className="text-xs text-muted-foreground">{t.gpuLayers}</label>
+                                    <Input
+                                        type="number"
+                                        min={0}
+                                        step={1}
+                                        placeholder={t.gpuLayersAuto}
+                                        value={params.gpuLayers ?? ""}
+                                        onChange={(e) =>
+                                            updateParam({ gpuLayers: e.target.value === "" ? undefined : Number(e.target.value) })
+                                        }
+                                        aria-label={t.gpuLayers}
+                                        title={t.gpuLayersHelp}
+                                        className="h-8 text-xs"
+                                    />
+                                </div>
                                 <div className="flex flex-col gap-1">
                                     <label className="text-xs text-muted-foreground">{t.frequencyPenalty}</label>
                                     <Input

frontend/src/lib/translations.ts

@@ -15,6 +15,9 @@ export interface Dictionary {
     attachFiles: string;
     attachProjectFolder: string;
     noOllamaModelsInstalled: string;
+    huggingFaceHint: string;
+    pullFromHuggingFace: string;
+    pullExactTag: string;
     agentMode: string;
     agentModeTooltip: string;
     changeFolder: string;
@@ -46,6 +49,9 @@ export interface Dictionary {
     topP: string;
     maxTokens: string;
     contextLength: string;
+    gpuLayers: string;
+    gpuLayersAuto: string;
+    gpuLayersHelp: string;
     contextLengthOllamaOnly: string;
     frequencyPenalty: string;
     presencePenalty: string;
@@ -98,6 +104,9 @@ export const en: Dictionary = {
     attachFiles: "Attach files",
     attachProjectFolder: "Attach project folder",
     noOllamaModelsInstalled: "No Ollama models installed — go to Settings to install one.",
+    huggingFaceHint: "Tip: paste any GGUF model's Hugging Face URL (or hf.co/user/repo) to pull it directly — you're not limited to the catalog above.",
+    pullFromHuggingFace: "Pull this GGUF model directly from Hugging Face.",
+    pullExactTag: "Not in the catalog — pull this exact model tag from Ollama's library.",
     agentMode: "Agent",
     agentModeTooltip: "Agent mode: gives the model file tools (read/write/list/search) and shell command execution, scoped to a folder you choose. Every tool call needs your approval.",
     changeFolder: "Change folder",
@@ -130,6 +139,9 @@ export const en: Dictionary = {
     topP: "Top P",
     maxTokens: "Max tokens",
     contextLength: "Context length",
+    gpuLayers: "GPU layers",
+    gpuLayersAuto: "Auto",
+    gpuLayersHelp: "How many model layers to offload to the GPU (Ollama's num_gpu). Leave blank to let Ollama decide automatically; 0 forces CPU-only.",
     contextLengthOllamaOnly: " (Ollama only)",
     frequencyPenalty: "Frequency penalty",
     presencePenalty: "Presence penalty",
@@ -183,6 +195,9 @@ export const tr: Dictionary = {
     attachFiles: "Dosya ekle",
     attachProjectFolder: "Proje klasörü ekle",
     noOllamaModelsInstalled: "Yüklü Ollama modeli yok — birini yüklemek için Ayarlar'a gidin.",
+    huggingFaceHint: "İpucu: herhangi bir GGUF modelinin Hugging Face bağlantısını (veya hf.co/kullanıcı/depo) yapıştırarak doğrudan indirebilirsiniz — yukarıdaki katalogla sınırlı değilsiniz.",
+    pullFromHuggingFace: "Bu GGUF modelini doğrudan Hugging Face'ten indir.",
+    pullExactTag: "Katalogda yok — bu tam model etiketini Ollama kütüphanesinden indirin.",
     agentMode: "Ajan",
     agentModeTooltip: "Ajan modu: modele seçtiğiniz bir klasörle sınırlı dosya araçları (okuma/yazma/listeleme/arama) ve kabuk komutu çalıştırma verir. Her araç çağrısı onayınızı gerektirir.",
     changeFolder: "Klasörü değiştir",
@@ -216,6 +231,9 @@ export const tr: Dictionary = {
     topP: "Top P",
     maxTokens: "Maksimum token",
     contextLength: "Bağlam uzunluğu",
+    gpuLayers: "GPU katmanları",
+    gpuLayersAuto: "Otomatik",
+    gpuLayersHelp: "GPU'ya kaç model katmanının aktarılacağı (Ollama'nın num_gpu ayarı). Ollama'nın otomatik karar vermesi için boş bırakın; 0 sadece CPU kullanımını zorlar.",
     contextLengthOllamaOnly: " (yalnızca Ollama)",
     frequencyPenalty: "Sıklık cezası",
     presencePenalty: "Varlık cezası",

frontend/src/pages/Chat.tsx

@@ -506,6 +506,7 @@ export default function Chat() {
                 params.frequencyPenalty ?? project?.params?.frequencyPenalty ?? settings?.frequencyPenalty,
             presencePenalty: params.presencePenalty ?? project?.params?.presencePenalty ?? settings?.presencePenalty,
             contextLength: params.contextLength ?? project?.params?.contextLength ?? settings?.contextLength,
+            gpuLayers: params.gpuLayers ?? project?.params?.gpuLayers ?? settings?.gpuLayers,
         };
     }
 
@@ -1026,6 +1027,25 @@ export default function Chat() {
                                     className="h-8 text-xs"
                                 />
                             </div>
+                            <div className="flex flex-col gap-1">
+                                <label className="text-xs text-muted-foreground">
+                                    {t.gpuLayers}
+                                    {parsedModel?.provider !== "ollama" ? " (Ollama only)" : ""}
+                                </label>
+                                <Input
+                                    type="number"
+                                    min={0}
+                                    step={1}
+                                    placeholder={t.gpuLayersAuto}
+                                    title={t.gpuLayersHelp}
+                                    value={params.gpuLayers ?? currentProject?.params?.gpuLayers ?? settings?.gpuLayers ?? ""}
+                                    onChange={(e) =>
+                                        updateParam({ gpuLayers: e.target.value === "" ? undefined : Number(e.target.value) })
+                                    }
+                                    disabled={parsedModel?.provider !== "ollama"}
+                                    className="h-8 text-xs"
+                                />
+                            </div>
                             <div className="flex flex-col gap-1">
                                 <label className="text-xs text-muted-foreground">Freq. penalty</label>
                                 <Input