From f3bb75398cf84637514af58038d19b364fb71444 Mon Sep 17 00:00:00 2001
From: Caleb de Leeuw <cdeleeuw@users.noreply.github.com>
Date: Tue, 24 Mar 2026 13:49:07 -0700
Subject: [PATCH] fix: check GPU VRAM against NIM model requirements before
 setting nimCapable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

detectGpu() unconditionally set nimCapable=true for any NVIDIA GPU,
even when no NIM model fits in the available VRAM. This caused
onboard to pass --gpu to gateway and sandbox creation, which fails
on systems where GPU passthrough is unavailable (e.g. WSL2) or
the GPU has insufficient memory for any local model.

Now nimCapable is only true when at least one NIM model in
nim-images.json has minGpuMemoryMB <= totalMemoryMB. When false,
onboard skips --gpu flags and auto-selects cloud inference,
with a clear message explaining why.

Tested on an 8GB VRAM GPU (RTX) where the smallest NIM model
requires 8192MB — onboard now completes successfully using
NVIDIA Cloud API.
---
 bin/lib/nim.js     | 4 +++-
 bin/lib/onboard.js | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/bin/lib/nim.js b/bin/lib/nim.js
index 4f2233e435f..a383fe0b021 100644
--- a/bin/lib/nim.js
+++ b/bin/lib/nim.js
@@ -35,12 +35,14 @@ function detectGpu() {
       const perGpuMB = lines.map((l) => parseInt(l.trim(), 10)).filter((n) => !isNaN(n));
       if (perGpuMB.length > 0) {
         const totalMemoryMB = perGpuMB.reduce((a, b) => a + b, 0);
+        // Only mark nimCapable if at least one NIM model fits in GPU VRAM
+        const canRunNim = nimImages.models.some((m) => m.minGpuMemoryMB <= totalMemoryMB);
         return {
           type: "nvidia",
           count: perGpuMB.length,
           totalMemoryMB,
           perGpuMB: perGpuMB[0],
-          nimCapable: true,
+          nimCapable: canRunNim,
         };
       }
     }
diff --git a/bin/lib/onboard.js b/bin/lib/onboard.js
index 3e845342562..401bcd11d8e 100644
--- a/bin/lib/onboard.js
+++ b/bin/lib/onboard.js
@@ -71,6 +71,9 @@ async function preflight() {
   const gpu = nim.detectGpu();
   if (gpu && gpu.type === "nvidia") {
     console.log(`  ✓ NVIDIA GPU detected: ${gpu.count} GPU(s), ${gpu.totalMemoryMB} MB VRAM`);
+    if (!gpu.nimCapable) {
+      console.log("  ⓘ GPU VRAM too small for local NIM — will use cloud inference");
+    }
   } else if (gpu && gpu.type === "apple") {
     console.log(`  ✓ Apple GPU detected: ${gpu.name}${gpu.cores ? ` (${gpu.cores} cores)` : ""}, ${gpu.totalMemoryMB} MB unified memory`);
     console.log("  ⓘ NIM requires NVIDIA GPU — will use cloud inference");