From 15e3b77357ed332368832ed9d36fcb7ac24067ad Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Mon, 10 Aug 2026 23:03:17 +0000 Subject: [PATCH 01/12] ggml-openvino: keep the rope angle in f32 under the f16 inference hint --- .../src/ggml-openvino/ggml-openvino-extra.cpp | 1 + ggml/src/ggml-openvino/openvino/utils.cpp | 70 +++++++++++++++++-- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 0257e23dbf3..d4f7ef299f8 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -136,6 +136,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_RELEASE_WEIGHTS", "GGML_OPENVINO_REDUCE_COMPILE_MEM", "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + "GGML_OPENVINO_ROPE_F32_ANGLE", "GGML_OPENVINO_LOG_SWA_LAYERS", "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 40ba938a62c..fbabcd3b0e0 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,5 +1,7 @@ #include "utils.h" +#include "../ggml-openvino-extra.h" + #include #include #include @@ -20,6 +22,7 @@ #include #include #include +#include #include namespace ov { @@ -317,14 +320,73 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params } } - Output cos_theta = std::make_shared(theta); - Output sin_theta = std::make_shared(theta); + // Keep the ROPE ANGLE in f32 even under INFERENCE_PRECISION_HINT=f16. + // + // Everything above builds theta = pos * freq_factors with a declared f32 type, but a declared + // element type is not an execution precision: under the f16 hint the GPU plugin compresses these + // subgraphs to f16 anyway. theta's magnitude IS the token position, so at position 6525 it lands + // where f16 spacing is 4 and the value is not even representable. Worse, a model whose freq_base + // and head_dim make factor[0] == 1.0 exactly feeds the raw position in as the top-frequency + // angle, so an absolute error of 1-2 becomes an error of 1-2 RADIANS -- up to a third of a cycle. Sin/Cos of an angle wrong by radians is not approximately + // right, it is arbitrary, and trig amplifies the error without bound. + // + // Measured on a large decoder-only model at inp_pos=6525, same device, f16 vs f32 execution: the + // whole Q/K projection chain is clean at rel ~1e-4, then Cos jumps to rel 0.124 and Sin to 0.155, + // and K right after the rotation carries rel 0.081. The error is depth-dependent because f16 + // spacing grows with magnitude: sin rel is 0.006 at position 512 and 0.16 at 6525. + // + // mark_as_precision_sensitive disables fp16 compression of the subgraph feeding an input, which is + // exactly the guarantee needed. Marking Sin/Cos's input covers the whole theta chain back to the + // Convert of inp_pos. The cost is negligible: theta is a few dozen values per token, against the + // matmuls that dominate. The rotation itself and everything downstream stay f16. + const bool pin_angle = ggml_openvino_getenv_int("GGML_OPENVINO_ROPE_F32_ANGLE", 1) != 0; + + auto cos_op = std::make_shared(theta); + auto sin_op = std::make_shared(theta); + if (pin_angle) { + ov::mark_as_precision_sensitive(cos_op->input(0)); + ov::mark_as_precision_sensitive(sin_op->input(0)); + } + Output cos_theta = cos_op; + Output sin_theta = sin_op; if (mscale != 1.0f) { auto mscale_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {mscale}); - cos_theta = std::make_shared(cos_theta, mscale_node); - sin_theta = std::make_shared(sin_theta, mscale_node); + auto cos_scaled = std::make_shared(cos_theta, mscale_node); + auto sin_scaled = std::make_shared(sin_theta, mscale_node); + // Mark this Constant's input too. Marking only Sin/Cos keeps those nodes f32 but leaves the + // sibling mscale Constant to be compressed to f16, and the pass does not reconcile the pair, + // so compilation fails with "Arguments do not have the same element type". + if (pin_angle) { + ov::mark_as_precision_sensitive(cos_scaled->input(1)); + ov::mark_as_precision_sensitive(sin_scaled->input(1)); + } + cos_theta = cos_scaled; + sin_theta = sin_scaled; + } + + // Close the f32 island explicitly with an f16 round trip. + // + // ConvertPrecision does not insert the boundary Convert: with only the marking above, the f32 + // region propagates forward through the shape-only ops of the cos/sin expansion and then collides + // with f16 data in the rotation, so the type mismatch just moves one step down the graph. + // + // Rounding the RESULT of the trig is harmless -- cos/sin are in [-1,1], where f16 spacing is ~6e-4, + // the ordinary f16 cost accepted everywhere else. What must not be rounded is the ANGLE, which the + // marking above protects. So this spends the cheap rounding to buy a type-consistent graph: the + // chain from inp_pos through Sin/Cos stays f32, everything downstream is f16, and the transition is + // a node we control rather than one the pass has to infer. + // + // Declared types are f32 on both sides at build time, so this is a no-op for the frontend and for + // the f32 paths; it only becomes a real boundary once ConvertPrecision runs under the f16 hint. + if (pin_angle) { + auto pin_boundary = [](Output x) -> Output { + auto down = std::make_shared(x, ov::element::f16); + return std::make_shared(down, ov::element::f32); + }; + cos_theta = pin_boundary(cos_theta); + sin_theta = pin_boundary(sin_theta); } return std::make_pair(sin_theta, cos_theta); From d09fd8b89e7d62251d3e64f2b67900df2ec1766d Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Mon, 10 Aug 2026 23:08:52 +0000 Subject: [PATCH 02/12] ggml-openvino: honour GGML_TENSOR_FLAG_OUTPUT when computing model outputs --- ggml/src/ggml-openvino/ggml-decoder.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index ae36a09591d..6c7b39cbc65 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1357,6 +1357,30 @@ void GgmlOvDecoder::compute_model_outputs() { if (::is_inplace_op(cur_node) && ggml_nbytes(cur_node) == 0) { continue; } + // A node explicitly marked by ggml_set_output() must be a model output even when it is + // fully consumed inside the graph, because the host reads it back afterwards. The + // use-count analysis below cannot see that: it only asks whether anything still in the + // graph needs the value. + // + // Mid-graph taps break that assumption. A model that exposes its residual stream at a few + // layers (llm_graph_result marks each such tensor with ggml_set_output()) has tensors that + // are read by the host AND feed the next layer, so use-count logic dropped them, the OV + // model never produced them, and the host read an unwritten buffer -- whatever consumes + // those features saw zeros. This affects any consumer of a mid-graph tensor, not one + // particular feature. + if (cur_node->flags & GGML_TENSOR_FLAG_OUTPUT) { + // in-place ops publish their result through view_src, as in the use_count == 0 path + auto * flagged_node = cur_node; + if (flagged_node->op == GGML_OP_SET_ROWS && flagged_node->view_src != nullptr) { + flagged_node = flagged_node->view_src; + } + std::string flagged_name(flagged_node->name); + if (m_model_outputs.find(flagged_name) == m_model_outputs.end()) { + m_model_outputs[flagged_name] = flagged_node; + m_model_output_names.insert(flagged_name); + } + continue; + } auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)]; if (cur_node_use_count == 0) { // The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src. From 6b1a11e780cc890c8b97f7a1e09dc05043230b21 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Mon, 10 Aug 2026 23:18:57 +0000 Subject: [PATCH 03/12] ggml-openvino: add a GGML_OP_ARGMAX translator --- ggml/src/ggml-openvino/openvino/op/argmax.cpp | 51 +++++++++++++++++++ ggml/src/ggml-openvino/openvino/op_table.cpp | 1 + ggml/src/ggml-openvino/openvino/op_table.h | 1 + 3 files changed, 53 insertions(+) create mode 100644 ggml/src/ggml-openvino/openvino/op/argmax.cpp diff --git a/ggml/src/ggml-openvino/openvino/op/argmax.cpp b/ggml/src/ggml-openvino/openvino/op/argmax.cpp new file mode 100644 index 00000000000..441a6a5b2ad --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/argmax.cpp @@ -0,0 +1,51 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML_OP_ARGMAX: src0 is a matrix [ne0, ne1]; the result is a 1-D I32 tensor of ne1 entries +// holding, for each row, the index of the maximum along ne0. ggml shapes arrive reversed here, so +// ne0 is the last OV axis and ne1 is axis 2. TopK(k=1, axis=last, MAX) yields exactly that index, +// then reshape to the [1,1,1,ne1] layout the decoder expects for a 1-D ggml output. +// +// Note ggml's argmax returns the FIRST maximum on a tie, while TopK's tie-breaking is not specified +// to match. Ties on real logits are vanishingly rare and none were observed, but the two are not +// proven equivalent. +OutputVector translate_argmax(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + + auto k = ov::op::v0::Constant::create(ov::element::i64, {}, {1}); + auto topk = std::make_shared(input, + k, + 3, + ov::op::v11::TopK::Mode::MAX, + ov::op::v11::TopK::SortType::SORT_VALUES, + context.get_output_type(), + false); + + // ne1 is the dynamic row count, so build the target shape at runtime. + auto leading = ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 1, 1}); + auto rows = get_dimensions(input.get_node_shared_ptr(), {2}); + auto target_shape = std::make_shared(OutputVector{leading, rows}, 0); + auto res = std::make_shared(topk->output(1), target_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 12a0953e3cf..d5bcd42028f 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -56,6 +56,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_SQR", op::translate_sqr }, {"GGML_OP_SQRT", op::translate_sqrt }, {"GGML_OP_SOFT_MAX", op::translate_soft_max }, + {"GGML_OP_ARGMAX", op::translate_argmax }, {"GGML_OP_ARGSORT", op::translate_argsort }, {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, {"GGML_OP_TRANSPOSE", op::translate_transpose }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index 6ed7549a265..4d073c327a6 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -42,6 +42,7 @@ GGML_OP_CONVERTER(translate_glu_geglu); GGML_OP_CONVERTER(translate_glu_geglu_quick); GGML_OP_CONVERTER(translate_set_rows); GGML_OP_CONVERTER(translate_cpy); +GGML_OP_CONVERTER(translate_argmax); GGML_OP_CONVERTER(translate_argsort); GGML_OP_CONVERTER(translate_flash_attn_ext); GGML_OP_CONVERTER(translate_clamp); From e20d6c074c6b10de664292a3b666e0ab4373a1d3 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Tue, 11 Aug 2026 00:51:25 +0000 Subject: [PATCH 04/12] ggml-openvino: refresh the decoder IO when the cgraph instance changes --- ggml/src/ggml-openvino/ggml-decoder.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 33b95340a1b..18e82295f40 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -382,6 +382,16 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM; } + // A 2-D float graph input feeding a matmul as src[1]. Speculative decoding hands the draft the + // target's hidden features this way ([n_embd, n_tokens], with no GET_ROWS lookup, so none of the + // other seeds match it) and the token count differs between the prompt pass and the shorter + // draft blocks, so its token dim must stay dynamic. + static bool is_inp_embd_2d(const ggml_tensor * tensor, const ggml_tensor * op) { + return (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->op == GGML_OP_NONE && + tensor->type == GGML_TYPE_F32 && tensor->ne[2] == 1 && tensor->ne[3] == 1 && + op->op == GGML_OP_MUL_MAT && tensor == op->src[1]; + } + static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); From c5aa179e631f191c09e8468b2c40dd611638bc84 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Tue, 11 Aug 2026 00:51:25 +0000 Subject: [PATCH 05/12] ggml-openvino: keep the token dim dynamic for a 2-D activation graph input --- ggml/src/ggml-openvino/ggml-decoder.cpp | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 6c7b39cbc65..0a995ec009f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -2090,6 +2090,17 @@ const std::string & GgmlOvDecoder::get_op_type() const { return unknown_op; } +namespace { +bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) { + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (a->ne[i] != b->ne[i]) { + return false; + } + } + return true; +} +} // namespace + void GgmlOvDecoder::compute_node_dynamic_dims() { auto visit_node = [&](auto && self, ggml_tensor * node) -> void { if (!node) { @@ -2129,6 +2140,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { m_node_dynamic_dims[src] = 1; continue; } + if (is_inp_embd_2d(src, node)) { + m_node_dynamic_dims[src] = 1; + continue; + } self(self, src); } } @@ -2184,6 +2199,11 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { } break; case GGML_OP_VIEW: { + // A VIEW that does not change the shape is a pass-through for dynamic-dim purposes. + if (is_same_shape(node->src[0], node)) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + } // Use stride-based matching: the stride of a VIEW dimension directly // encodes which source dimension it indexes into, so it uniquely // identifies the dynamic dim even when two dims share the same size. @@ -2260,6 +2280,19 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { } break; } + case GGML_OP_CONCAT: + // The concatenated axis is the one whose extent changed. + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->src[0]->ne[i] != node->ne[i]) { + m_node_dynamic_dims[node] = i; + break; + } + } + break; + case GGML_OP_SSM_CONV: + case GGML_OP_GATED_DELTA_NET: + m_node_dynamic_dims[node] = 1; + break; case GGML_OP_CONT: m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[0]] != -1) { From 25dc71f8ec0dfef9dd5bb83a9d9f6f9f9cbbb605 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Tue, 11 Aug 2026 02:05:05 +0000 Subject: [PATCH 06/12] ggml-openvino: tolerate graph inputs and outputs with no backing storage --- ggml/src/ggml-openvino/utils.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 67a4123cb19..fb92f90b62b 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -96,6 +96,12 @@ std::optional try_make_kv_sliced_tensor(const std::shared_ptrdata == nullptr) { + // unallocated input: let the caller build an owning zero tensor rather than slice a null + // pointer + return std::nullopt; + } + ov::Shape sliced_shape = full_shape; sliced_shape[2] = static_cast(n_kv); @@ -416,6 +422,11 @@ ov::Tensor create_ov_output_tensor(const std::shared_ptr & ggml_d } } + // A graph output can also have no backing storage. Give OV an owning tensor so the infer request + // has somewhere to write; nothing downstream reads it. + if (output_data == nullptr) { + return ov::Tensor(output_type, output_shape); + } ov::Tensor output_tensor(output_type, output_shape, output_data); return output_tensor; } @@ -509,6 +520,20 @@ ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_ return make_contiguous_split_input_tensor(ggml_tensor, input_shape); } + // A graph input may legitimately be left unallocated (data == nullptr, no buffer): llama.cpp + // skips filling the attention mask when a graph only stores K/V without attending, which is what + // a speculative KV-injection pass does. Such an input is never read, but OpenVINO still requires + // a backing tensor for the Parameter and constructing an ov::Tensor over a null pointer asserts. + // Hand OV an owning zero-filled tensor; its contents cannot affect the result because no op + // consumes them. + if (input_data == nullptr) { + GGML_LOG_WARN("ggml-openvino: unallocated INPUT '%s' (ggml name '%s', type %s, ne=[%ld,%ld,%ld,%ld]) " + "-> zero tensor\n", + name.c_str(), ggml_tensor->name, ggml_type_name(ggml_tensor->type), + (long) ggml_tensor->ne[0], (long) ggml_tensor->ne[1], (long) ggml_tensor->ne[2], + (long) ggml_tensor->ne[3]); + return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + } auto input_tensor = ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape, input_data); return input_tensor; } From c174406060937636f2b0853b586b1b62a530b817 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Tue, 11 Aug 2026 03:36:21 +0000 Subject: [PATCH 07/12] ggml-openvino: invalidate a cached graph when its output set changes --- ggml/src/ggml-openvino/ggml-decoder.cpp | 21 +++++++++++++++++++++ ggml/src/ggml-openvino/ggml-decoder.h | 7 +++++++ ggml/src/ggml-openvino/utils.cpp | 9 +++++++++ 3 files changed, 37 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 0a995ec009f..dffb7201236 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -35,6 +35,24 @@ #include #include +static std::set collect_graph_output_names(const ggml_cgraph * cgraph) { + std::set outputs; + for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) { + auto * node = cgraph->nodes[node_n]; + if (node->flags & GGML_TENSOR_FLAG_OUTPUT) { + outputs.insert(node->name); + } + } + return outputs; +} + +bool GgmlOvDecoder::has_same_graph_io(const ggml_cgraph * cgraph) const { + if (m_built_output_names.empty()) { + return true; // no snapshot (naive / test decoders): nothing to compare against + } + return collect_graph_output_names(cgraph) == m_built_output_names; +} + GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, ModelParams & model_params, ComputeParams & compute_params, @@ -64,6 +82,8 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, validate_cgraph(); + m_built_output_names = collect_graph_output_names(cgraph); + set_input_output(); compute_node_dynamic_dims(); compute_model_inputs(); @@ -79,6 +99,7 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, void GgmlOvDecoder::update_io(ggml_cgraph * cgraph) { m_cgraph = cgraph; + m_built_output_names = collect_graph_output_names(cgraph); m_model_inputs.clear(); m_model_outputs.clear(); m_node_info_list.clear(); diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 18e82295f40..052a4650451 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -363,6 +363,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void update_io(ggml_cgraph * cgraph); bool is_bound_to(const ggml_cgraph * cgraph) const; + // Was this decoder built for a graph with the same OUTPUT set as `cgraph`? The cache key is + // only (n_nodes, first/last node name), which cannot see a change in which tensors are marked + // as outputs, and the compiled model's Results are fixed at compile time. + bool has_same_graph_io(const ggml_cgraph * cgraph) const; + static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; } @@ -502,6 +507,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { std::map> m_model_weights; std::map m_model_outputs; std::set m_model_output_names; + // The output set the compiled model was built from, for has_same_graph_io(). + std::set m_built_output_names; std::vector m_node_info_list; std::map m_node_dynamic_dims; diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index fb92f90b62b..de7d97eba4e 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -922,6 +922,15 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share if (cache_hit) { ggml_decoder = entry->ptr; + // The cache key is only (n_nodes, first/last node name), so it cannot see a change in + // WHICH tensors are graph outputs -- and the compiled model's Results are fixed when it is + // built. A consumer that starts requesting extra outputs later (speculative decoding marks + // the target's per-layer residuals only once the draft is initialised) would otherwise keep + // reusing a model that never produces them, and the host would read unwritten buffers as + // zeros. Recompile when the output set differs. + if (!ggml_decoder->has_same_graph_io(cgraph)) { + cache_hit = false; + } old_m_params = ggml_decoder->get_model_params(); if (!ggml_decoder->is_splited_model()) { cache_hit = old_m_params.can_reuse_dynamically(m_params); From c64f8332d2b986966f85946570969423b0e3c6b7 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Tue, 11 Aug 2026 04:40:42 +0000 Subject: [PATCH 08/12] ggml-openvino: apply the output-set check after can_reuse_dynamically --- ggml/src/ggml-openvino/utils.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index de7d97eba4e..3b64e4e54ae 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -928,13 +928,15 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share // the target's per-layer residuals only once the draft is initialised) would otherwise keep // reusing a model that never produces them, and the host would read unwritten buffers as // zeros. Recompile when the output set differs. - if (!ggml_decoder->has_same_graph_io(cgraph)) { - cache_hit = false; - } old_m_params = ggml_decoder->get_model_params(); if (!ggml_decoder->is_splited_model()) { cache_hit = old_m_params.can_reuse_dynamically(m_params); } + // Must come AFTER can_reuse_dynamically(), which ASSIGNS to cache_hit rather than + // and-ing into it and would otherwise silently undo this invalidation. + if (cache_hit && !ggml_decoder->has_same_graph_io(cgraph)) { + cache_hit = false; + } } std::vector ov_input_names; From e54ae10a226dd6f842773bc729182478aa829747 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Thu, 3 Sep 2026 22:02:10 +0200 Subject: [PATCH 09/12] ggml-openvino: fix build+correctness bugs from the PR #26888 auto-merge --- ggml/src/ggml-openvino/ggml-decoder.cpp | 26 +------------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index dffb7201236..3007ba2216b 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1395,7 +1395,7 @@ void GgmlOvDecoder::compute_model_outputs() { if (flagged_node->op == GGML_OP_SET_ROWS && flagged_node->view_src != nullptr) { flagged_node = flagged_node->view_src; } - std::string flagged_name(flagged_node->name); + std::string flagged_name = get_tensor_ov_name(m_cgraph, flagged_node); if (m_model_outputs.find(flagged_name) == m_model_outputs.end()) { m_model_outputs[flagged_name] = flagged_node; m_model_output_names.insert(flagged_name); @@ -2111,17 +2111,6 @@ const std::string & GgmlOvDecoder::get_op_type() const { return unknown_op; } -namespace { -bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) { - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (a->ne[i] != b->ne[i]) { - return false; - } - } - return true; -} -} // namespace - void GgmlOvDecoder::compute_node_dynamic_dims() { auto visit_node = [&](auto && self, ggml_tensor * node) -> void { if (!node) { @@ -2301,19 +2290,6 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { } break; } - case GGML_OP_CONCAT: - // The concatenated axis is the one whose extent changed. - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->src[0]->ne[i] != node->ne[i]) { - m_node_dynamic_dims[node] = i; - break; - } - } - break; - case GGML_OP_SSM_CONV: - case GGML_OP_GATED_DELTA_NET: - m_node_dynamic_dims[node] = 1; - break; case GGML_OP_CONT: m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[0]] != -1) { From 91172c0002eb77d6c6bf46ae8957eef04560d75a Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Thu, 3 Sep 2026 23:29:23 +0200 Subject: [PATCH 10/12] ggml-openvino: bound the compiled-model cache to N shapes instead of 1 --- ggml/src/ggml-openvino/utils.cpp | 6 +++-- ggml/src/ggml-openvino/utils.h | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 3b64e4e54ae..fd84d391e2b 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -904,11 +904,12 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share cache_hit = it != r_ctx->decoder_cache.end(); if (cache_hit) { entry = it->second; + r_ctx->touch_locked(key); } else { - r_ctx->clear_caches_locked(); auto mutex = std::make_shared(); entry = std::make_shared(mutex); r_ctx->decoder_cache[key] = entry; + r_ctx->admit_locked(key); } } else { auto mutex = std::make_shared(); @@ -1494,11 +1495,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared cache_hit = it != r_ctx->decoder_cache.end(); if (cache_hit) { entry = it->second; + r_ctx->touch_locked(key); } else { - r_ctx->clear_caches_locked(); auto mutex = std::make_shared(); entry = std::make_shared(mutex); r_ctx->decoder_cache[key] = entry; + r_ctx->admit_locked(key); } } else { auto mutex = std::make_shared(); diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 491dbadc2e9..4d4b23e52b2 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -117,17 +118,55 @@ struct ov_runtime_context { std::unordered_map, graph_key_hash> infer_request_cache_prefill; std::unordered_map, graph_key_hash> ov_input_names_cache; std::unordered_map, graph_key_hash> ov_output_names_cache; + // LRU order of decoder_cache's keys, oldest (least recently used) at the front. A pipeline + // that interleaves distinct graph shapes every call -- e.g. DFlash cycling target/draft/ + // injection graphs every round -- needs a slot per shape open at once; a single-entry cache + // would evict and recompile all of them, every call, forever. + std::list lru_keys; + static constexpr size_t max_cached_graphs = 8; size_t stateful_kv_size; std::map kv_state_input_name_map; ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0) {} + // Mark `key` as the most recently used entry. Call on every cache hit and after inserting a + // new entry, so eviction always picks the graph shape that has gone longest unused. + void touch_locked(const graph_key & key) { + lru_keys.remove(key); + lru_keys.push_back(key); + } + + // Drop the least recently used graph shape's compiled model and its associated per-key + // caches. Leaves every other shape's compiled model intact. + void evict_lru_locked() { + if (lru_keys.empty()) { + return; + } + graph_key victim = lru_keys.front(); + lru_keys.pop_front(); + decoder_cache.erase(victim); + infer_request_cache.erase(victim); + infer_request_cache_prefill.erase(victim); + ov_input_names_cache.erase(victim); + ov_output_names_cache.erase(victim); + } + + // Make room for one more graph shape if the cache is at capacity, then mark `key` as most + // recently used. Call right after inserting `key` into decoder_cache. + void admit_locked(const graph_key & key) { + while (decoder_cache.size() > max_cached_graphs) { + evict_lru_locked(); + } + touch_locked(key); + } + void clear_caches_locked() { decoder_cache.clear(); infer_request_cache.clear(); infer_request_cache_prefill.clear(); ov_input_names_cache.clear(); ov_output_names_cache.clear(); + lru_keys.clear(); kv_state_input_name_map.clear(); stateful_kv_size = 0; } From 29a8866993fe694ea28bf47161552e32f36cbbef Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Tue, 18 Aug 2026 22:12:33 +0000 Subject: [PATCH 11/12] ggml-openvino: add GGML_OPENVINO_LOG_SUPPORTS_OP diagnostic --- .../src/ggml-openvino/ggml-openvino-extra.cpp | 1 + ggml/src/ggml-openvino/ggml-openvino.cpp | 103 +++++++++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index d4f7ef299f8..e51f76bfd8a 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -144,6 +144,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", // Build the precise (but O(n_nodes)) graph cache key. Needed by op tests. "GGML_OPENVINO_FULL_GRAPH_KEY", + "GGML_OPENVINO_LOG_SUPPORTS_OP", }; for (const char * const & env_var : env_var_names) { diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 87ee096fa54..9b5c32f3d4b 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -1870,16 +1871,114 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg return {true, ""}; } +// Report what supports_op() accepts and rejects, so a graph that silently falls back to another +// backend can be diagnosed. Every op rejected here becomes a graph split; if nothing is rejected, +// the whole graph runs on OpenVINO. +// +// Gated by GGML_OPENVINO_LOG_SUPPORTS_OP: +// 1 one summary at exit listing every distinct op kind and verdict -- start here +// 2 additionally log every individual call, with the tensor name and shape +// +// This wraps the decision function instead of instrumenting it, so all of its return paths are +// covered -- including the several that reject without logging a reason of their own, and any +// added later. +// +// The summary is reported at exit rather than on each op's first occurrence, because the earliest +// calls happen before the log handler emits anything: with level 2 the first line to actually +// appear already reads "call 1244". A first-occurrence line would therefore be silently dropped +// for exactly the ops that are only ever seen early. Counting is unaffected, so the summary is the +// reliable view; treat the level-2 stream as a sample, not a complete trace. +namespace { + +struct supports_op_tally { + std::mutex mutex; + // key: ":ok" / ":no" -> number of calls + std::map counts; + + // Report totals once, at exit. Registered lazily so a run with the knob off costs nothing. + // atexit runs while the ggml log callback is still valid, which static destruction does not + // guarantee. + void arm_summary() { + static std::once_flag armed; + std::call_once(armed, [] { + std::atexit([] { + auto & self = instance(); + std::lock_guard lock(self.mutex); + + size_t n_accept_kinds = 0; + size_t n_reject_kinds = 0; + GGML_LOG_WARN("ov-supports-op: ---- summary (distinct op kinds, with call counts) ----\n"); + for (const auto & [key, count] : self.counts) { + const bool ok = key.size() > 3 && key.compare(key.size() - 3, 3, ":ok") == 0; + ok ? ++n_accept_kinds : ++n_reject_kinds; + GGML_LOG_WARN("ov-supports-op: %-8s %-24s %zu call(s)\n", ok ? "ACCEPT" : "REJECT", + key.substr(0, key.size() - 3).c_str(), count); + } + if (n_reject_kinds == 0) { + GGML_LOG_WARN("ov-supports-op: %zu op kind(s) accepted, 0 rejected" + " -- every op was supported, so no split originated here\n", + n_accept_kinds); + } else { + GGML_LOG_WARN("ov-supports-op: %zu op kind(s) accepted, %zu REJECTED" + " -- each rejected kind forces a graph split\n", + n_accept_kinds, n_reject_kinds); + } + }); + }); + } + + static supports_op_tally & instance() { + static supports_op_tally tally; + return tally; + } +}; + +} // namespace + static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { auto res = ggml_backend_openvino_device_supports_op_impl(dev, op); - if (!res.is_supported) { + const bool supported = res.is_supported; + if (!supported) { static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0; if (log_unsupported) { GGML_LOG_WARN("OpenVINO op unsupported: op '%s' (%s), type %s: %s\n", op->name, ggml_op_name(op->op), ggml_type_name(op->type), res.reason.c_str()); } } - return res.is_supported; + + static const int log_level = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SUPPORTS_OP"); + if (log_level <= 0) { + return supported; + } + + // Name the op the way ggml does: UNARY and GLU carry their real identity in a sub-enum, so + // logging only "UNARY" would merge SILU with everything else. + std::string op_desc = ggml_op_name(op->op); + if (op->op == GGML_OP_UNARY) { + op_desc += std::string("/") + ggml_unary_op_name(ggml_get_unary_op(op)); + } else if (op->op == GGML_OP_GLU) { + op_desc += std::string("/") + ggml_glu_op_name(ggml_get_glu_op(op)); + } + + // The scheduler calls supports_op() on every graph build and may do so from several threads, + // so the tally needs a lock. Keyed on op identity *and* verdict, because the same op can be + // accepted at one shape and rejected at another (e.g. a quantized 3-D src). + auto & tally = supports_op_tally::instance(); + tally.arm_summary(); + + size_t count; + { + std::lock_guard lock(tally.mutex); + count = ++tally.counts[op_desc + (supported ? ":ok" : ":no")]; + } + + if (log_level >= 2) { + GGML_LOG_WARN("ov-supports-op: %-6s %-22s name='%s' type=%s ne=[%ld,%ld,%ld,%ld] (call %zu)\n", + supported ? "ACCEPT" : "REJECT", op_desc.c_str(), op->name, ggml_type_name(op->type), + (long) op->ne[0], (long) op->ne[1], (long) op->ne[2], (long) op->ne[3], count); + } + + return supported; } static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { From f3aec9e55fc7833029ca2beb8218b260ae1bfc61 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Fri, 4 Sep 2026 16:11:54 +0200 Subject: [PATCH 12/12] ggml-openvino: honour a node's dynamic-dim classification on non-split graphs too --- ggml/src/ggml-openvino/ggml-decoder.cpp | 2 +- .../src/ggml-openvino/ggml-openvino-extra.cpp | 1 + ggml/src/ggml-openvino/utils.cpp | 47 +++++++++++++++++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 3007ba2216b..9ea8e89008e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1176,7 +1176,7 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, } else { input_shape = ov::PartialShape{get_shape(input)}; } - if (dynamic_dim_index != -1 && m_model_is_splitted) { + if (dynamic_dim_index != -1) { input_shape[3 - dynamic_dim_index] = -1; } if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index e51f76bfd8a..d0cc099d81f 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -145,6 +145,7 @@ void ggml_openvino_device_config::init() { // Build the precise (but O(n_nodes)) graph cache key. Needed by op tests. "GGML_OPENVINO_FULL_GRAPH_KEY", "GGML_OPENVINO_LOG_SUPPORTS_OP", + "GGML_OPENVINO_DIAG_GRAPH_IO", }; for (const char * const & env_var : env_var_names) { diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index fd84d391e2b..31bb76d696b 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -1348,7 +1348,20 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share for (size_t i = 0; i < ov_input_names.size(); i++) { const auto & param_name = ov_input_names[i]; auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); + try { + infer_request->set_input_tensor(i, input_tensor); + } catch (const std::exception & e) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DIAG_GRAPH_IO")) { + auto port = infer_request->get_compiled_model().input(i); + GGML_LOG_WARN( + "ggml-openvino: DIAG set_input_tensor[%zu] '%s' threw for n_nodes=%d first=%s last=%s: %s\n" + " model_shape=%s bound_shape=%s\n", + i, param_name.c_str(), key.n_nodes, key.first_node_name.c_str(), key.last_node_name.c_str(), + e.what(), port.get_partial_shape().to_string().c_str(), + input_tensor.get_shape().to_string().c_str()); + } + throw; + } if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { print_input_tensor_info(param_name, input_tensor); @@ -1371,11 +1384,39 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share continue; } auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); + try { + infer_request->set_output_tensor(i, output_tensor); + } catch (const std::exception & e) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DIAG_GRAPH_IO")) { + auto port = infer_request->get_compiled_model().output(i); + GGML_LOG_WARN( + "ggml-openvino: DIAG set_output_tensor[%zu] '%s' threw for n_nodes=%d first=%s last=%s: %s\n" + " model_shape=%s bound_shape=%s\n", + i, ov_output_names[i].c_str(), key.n_nodes, key.first_node_name.c_str(), + key.last_node_name.c_str(), e.what(), port.get_partial_shape().to_string().c_str(), + output_tensor.get_shape().to_string().c_str()); + } + throw; + } } ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); + try { + infer_request->infer(); + } catch (const std::exception & e) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DIAG_GRAPH_IO")) { + auto cm = infer_request->get_compiled_model(); + GGML_LOG_WARN("ggml-openvino: DIAG infer() threw for n_nodes=%d first=%s last=%s: %s\n", key.n_nodes, + key.first_node_name.c_str(), key.last_node_name.c_str(), e.what()); + for (size_t i = 0; i < ov_input_names.size(); i++) { + auto port = cm.input(i); + GGML_LOG_WARN(" param[%zu] name=%s model_shape=%s bound_shape=%s\n", i, + ov_input_names[i].c_str(), port.get_partial_shape().to_string().c_str(), + infer_request->get_input_tensor(i).get_shape().to_string().c_str()); + } + } + throw; + } infer_end_time = ggml_time_us(); if (stateful_recurrent) { const auto * inp_pos = get_inp_pos_tensor(cgraph);