diff --git a/llama/compat/801-clip-node-stats-meter.patch b/llama/compat/801-clip-node-stats-meter.patch new file mode 100644 index 00000000000..b20caba39e6 --- /dev/null +++ b/llama/compat/801-clip-node-stats-meter.patch @@ -0,0 +1,53 @@ +diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp +index b9dd5e845..a24583e85 100644 +--- a/tools/mtmd/clip.cpp ++++ b/tools/mtmd/clip.cpp +@@ -222,7 +224,47 @@ struct clip_ctx { + ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false, true) + ); + +- if (ctx_params.cb_eval != nullptr) { ++ if (const char * flt = std::getenv("OLLAMA_CLIP_NODE_STATS")) { ++ // MAXUSAI (diagnostic, not for shipping): continuous per-node meter. ++ // Reports max|x| and how much of the tensor sits near the fp16 cliff ++ // for every node whose name contains the filter, with no early stop, ++ // so a run under GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32 measures how much ++ // headroom an image leaves before fp16 accumulation would overflow. ++ static std::string filter = flt; ++ ggml_backend_sched_set_eval_callback(sched.get(), ++ [](struct ggml_tensor * t, bool ask, void * /*ud*/) -> bool { ++ if (ask) { ++ return t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_F16; ++ } ++ const char * nm_c = ggml_get_name(t); ++ const std::string nm = nm_c ? nm_c : ""; ++ if (filter != "*" && nm.find(filter) == std::string::npos) { ++ return true; ++ } ++ const size_t n = ggml_nelements(t); ++ std::vector raw(ggml_nbytes(t)); ++ ggml_backend_tensor_get(t, raw.data(), 0, raw.size()); ++ float mx = 0.0f; ++ size_t n_inf = 0, n_nan = 0, n32 = 0, n49 = 0, n60 = 0; ++ for (size_t i = 0; i < n; i++) { ++ const float x = (t->type == GGML_TYPE_F32) ++ ? ((const float *) raw.data())[i] ++ : ggml_fp16_to_fp32(((const ggml_fp16_t *) raw.data())[i]); ++ if (std::isnan(x)) { n_nan++; continue; } ++ if (std::isinf(x)) { n_inf++; continue; } ++ const float a = std::fabs(x); ++ if (a > mx) mx = a; ++ if (a > 32752.0f) n32++; ++ if (a > 49128.0f) n49++; ++ if (a > 60000.0f) n60++; ++ } ++ LOG_ERR("CLIP_NODE_STATS name=%s op=%s type=%s n=%zu max_abs=%.1f " ++ "hr=%.4f n_gt32k=%zu n_gt49k=%zu n_gt60k=%zu n_inf=%zu n_nan=%zu\n", ++ nm.c_str(), ggml_op_name(t->op), ggml_type_name(t->type), ++ n, mx, mx / 65504.0f, n32, n49, n60, n_inf, n_nan); ++ return true; ++ }, nullptr); ++ } else if (ctx_params.cb_eval != nullptr) { + ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data); + } + diff --git a/llama/compat/README.md b/llama/compat/README.md index 490aeb0d815..5e0881cc876 100644 --- a/llama/compat/README.md +++ b/llama/compat/README.md @@ -42,6 +42,26 @@ intentionally skipped so a developer can iterate on a local llama.cpp tree. floor just under min instead. The budget is a hard ceiling; measured overshoots: nemotron pinned 3328 delivered 3388, gemma4 pinned 1120 delivered 1170 (pre-004). Affects only the infeasible pinned case. +- `801-clip-node-stats-meter.patch` - **not a compatibility shim; a diagnostic** + (see "Number bands" below). Adds an env-gated per-node meter to the clip + graph: with `OLLAMA_CLIP_NODE_STATS=` (or `*`) every matching + node's output is scanned after evaluation and its max |value|, counts above + 32k/49k/60k, and non-finite counts are logged. **Inert unless the variable is + set** - the eval callback is not even registered otherwise, so a normal build + and a normal run are unchanged. + + It exists because the qwen2.5-vl fp16-accumulate fault + (`docs/maxusai/qwen25vl-3b-poison-image-garbage-decode.md`, upstream + ollama/ollama#18070) is invisible from the product surface: the observable is + garbled text, the cause is a handful of elements out of millions overflowing + at `v.blk.31.ffn_down`. Diagnosing it without an instrument degenerates into + sampling, which cannot distinguish "no fault" from "not found yet". Run it + under `GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32` to read magnitudes without + overflowing; `max_abs / 65504` is the fp16 headroom. Calibrate against a known + trigger before trusting any run - an uncalibrated meter fails silently rather + than loudly. Usage and the trigger corpus: + `docs/maxusai/vision-suite/synthetic-triggers/README.md`. + - `903-fix-mmq-ids-padding.patch` - **not a compatibility shim** (see "Number bands" below). Sizes the MMQ ids-path tail padding in `ggml/src/ggml-cuda/mmq.cu` from the flattened row count @@ -71,22 +91,30 @@ fetched llama.cpp targets. The patch file only adds call sites. ### Number bands `apply-patch.cmake` globs `*.patch` recursively and applies them in sorted -filename order, so the numeric prefix is the apply order. Two bands share that -sequence and they mean different things: +filename order, so the numeric prefix is the apply order. Three bands share +that sequence and they mean different things: - **0xx — the compatibility layer.** Translating existing published Ollama GGUFs onto what llama.cpp already expects, plus the `models/` patches that register architectures llama.cpp does not have yet. These leave when the published models do. +- **8xx — diagnostics and instrumentation.** Env-gated observation code that + is inert in a normal build and run: nothing is registered and no behaviour + changes unless the operator sets the variable. They exist because some faults + are not observable from the product surface, and they stay as long as the + subsystem is worth being able to inspect. Unlike 0xx and 9xx they are not + waiting on anything, so they are **not** expected to leave. - **9xx — fork-carried fixes for defects in stock llama.cpp/ggml.** Not compatibility work: upstream-reportable bugs we are carrying until upstream takes the fix. They sort last so they apply on top of the compat layer, and they leave when the pinned `LLAMA_CPP_VERSION` moves past the fix. The distinction matters when a patch fails to apply after a llama.cpp bump. A -0xx failure means the insertion point moved and the patch needs regenerating. A -9xx failure often means upstream fixed the defect, and the right response is to -**delete the patch**, not re-cut it — check before regenerating. +0xx failure means the insertion point moved and the patch needs regenerating. An +8xx failure means the same — regenerate against the new anchor; the diagnostic +is not tracking an upstream fix. A 9xx failure often means upstream fixed the +defect, and the right response is to **delete the patch**, not re-cut it — +check before regenerating. ## Load-Time Hooks