Skip to content

QVAC-25534 fix: restore OpenVINO Windows CI #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
yingying0906 wants to merge 12 commits into
base: master
Choose a base branch
from
Draft

QVAC-25534 fix: restore OpenVINO Windows CI #310

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
651e0a6
QVAC-25534 fix: restore OpenVINO Windows tests
yingying0906 Sep 25, 2026
55d8618
QVAC-25534 fix: restore Windows OpenVINO coverage
yingying0906 Sep 25, 2026
9a62298
QVAC-25534 fix: keep MXFP4 MoE experts compressed on OpenVINO CPU
yingying0906 Sep 25, 2026
40e0a40
QVAC-25534 fix: limit MXFP4 path change to CPU
yingying0906 Sep 28, 2026
6b58c65
QVAC-25534 doc: clarify packed MXFP4 device paths
yingying0906 Sep 28, 2026
0153440
QVAC-25534 doc: remove unproven MSVC attribution
yingying0906 Sep 28, 2026
21ab94c
QVAC-25534 fix: bound OpenVINO MXFP4 CPU gather buffers
yingying0906 Sep 29, 2026
f5516cf
QVAC-25534 fix: chunk real OpenVINO CPU MXFP4 prefill
yingying0906 Oct 1, 2026
6aec67c
QVAC-25534 fix: handle changing OpenVINO output shapes
yingying0906 Oct 1, 2026
16346ba
QVAC-25534 fix: cap OpenVINO CPU MXFP4 MUL_MAT_ID instead of chunking it
yingying0906 Oct 5, 2026
14a3c52
QVAC-25534 fix: revert the OpenVINO CPU MXFP4 MUL_MAT_ID cap
yingying0906 Oct 5, 2026
42957c3
QVAC-25534 fix: keep out_ids and split OpenVINO views dynamic
yingying0906 Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/build-openvino.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ on:
types: [opened, synchronize, reopened]
paths: [
'.github/workflows/build-openvino.yml',
'common/fit.cpp',
'tests/test-fit-params.cpp',
'ggml/src/ggml-openvino/**'
]

Expand Down
43 changes: 38 additions & 5 deletions common/fit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@

#include <array>
#include <algorithm>
#include <atomic>
#include <cassert>
#include <stdexcept>
#include <cinttypes>
#include <exception>
#include <set>
#include <mutex>
#include <string>
Expand Down Expand Up @@ -47,6 +49,33 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
void * user_data;
} original_logger;
ggml_log_level min_level; // prints below this log level go to debug log
std::exception_ptr callback_exception;
std::mutex callback_exception_mutex;
std::atomic<bool> callback_failed = false;

// Keep callback exceptions inside the wrapper, then rethrow after the llama call so the logger guard restores.
void log(ggml_log_level level, const char * text) {
if (callback_failed.load(std::memory_order_acquire)) {
return;
}
try {
const ggml_log_level level_eff = level >= min_level ? level : GGML_LOG_LEVEL_DEBUG;
original_logger.callback(level_eff, text, original_logger.user_data);
} catch (...) {
const std::lock_guard<std::mutex> lock(callback_exception_mutex);
if (!callback_exception) {
callback_exception = std::current_exception();
}
callback_failed.store(true, std::memory_order_release);
}
}

void rethrow_callback_exception() {
const std::lock_guard<std::mutex> lock(callback_exception_mutex);
if (callback_exception) {
std::rethrow_exception(callback_exception);
}
}

~user_data_t() {
llama_log_set(original_logger.callback, original_logger.user_data);
Expand All @@ -56,23 +85,23 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
ud.min_level = log_level;

llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
const user_data_t * ud = (const user_data_t *) user_data;
const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
}, &ud);
llama_log_set([](ggml_log_level level, const char * text,
void * user_data) { ((user_data_t *) user_data)->log(level, text); },
&ud);

llama_model_params mparams_copy = *mparams;
mparams_copy.no_alloc = true;
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;

llama_model_ptr model_owner(llama_model_load_from_file(path_model, mparams_copy));
ud.rethrow_callback_exception();
llama_model * model = model_owner.get();
if (model == nullptr) {
throw std::runtime_error("failed to load model");
}

llama_context_ptr ctx_owner(llama_init_from_model(model, *cparams));
ud.rethrow_callback_exception();
llama_context * ctx = ctx_owner.get();
if (ctx == nullptr) {
throw common_params_fit_exception("failed to create llama_context from model");
Expand Down Expand Up @@ -108,6 +137,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
{
ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (cpu_dev == nullptr) {
ud.rethrow_callback_exception();
throw std::runtime_error("no CPU backend found");
}
size_t free;
Expand Down Expand Up @@ -153,6 +183,9 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
hp_n_expert = llama_model_n_expert(model);

common_memory_breakdown_print(ctx);
ctx_owner.reset();
model_owner.reset();
ud.rethrow_callback_exception();

return ret;
}
Expand Down
19 changes: 14 additions & 5 deletions ggml/src/ggml-openvino/ggml-decoder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,10 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph,
bool is_stateful,
bool model_is_splitted,
bool is_prefill,
int prefill_chunk_size) :
int prefill_chunk_size,
bool fixed_token_shape) :
m_is_static(is_static),
m_fixed_token_shape(fixed_token_shape),
m_is_stateful(is_stateful),
m_is_prefill(is_prefill),
m_naive(false),
Expand Down Expand Up @@ -694,7 +696,7 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,

if (is_inp_tok(input, op) || is_inp_pos(input, op)) {
// tokens or positions
int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : m_fixed_token_shape ? input->ne[0] : -1;
if (m_is_static && is_inp_pos(input, op)) {
// IMROPE stacks n_planes (t/h/w/e) position planes back to back
len *= get_inp_pos_n_planes(op);
Expand All @@ -713,6 +715,9 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
input_shape = ov::PartialShape{1, 1, -1, -1};
} else {
input_shape = ov::PartialShape{-1, 1, -1, -1};
if (m_fixed_token_shape) {
input_shape[2] = input->ne[1];
}
}

} else if (is_kvcache(input, op)) {
Expand All @@ -734,7 +739,7 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,

} else if (is_kv_idx(input, op)) {
// kv update index
int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : m_fixed_token_shape ? input->ne[0] : -1;
input_shape = ov::PartialShape{1, 1, 1, len};

} else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) {
Expand Down Expand Up @@ -1427,7 +1432,9 @@ ov::PartialShape GgmlOvDecoder::get_view_input_ov_shape(int node_idx,
if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) {
int dynamic_dim_index = dynamic_it->second;
// GGML uses reverse indexing, so convert to OpenVINO indexing
shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1;
shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() :
m_fixed_token_shape ? tensor->ne[dynamic_dim_index] :
-1;

@iancris iancris Oct 7, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fixed-token mode get_view_input_ov_shape turns every tracked dynamic dim into tensor->ne[dim], including dims rooted at inp_out_ids, whose graph input is still -1, so views downstream of an in-layer out_ids gather no longer match and get re-sliced.

compute_node_dynamic_dims tracks inp_out_ids as a dynamic dim (ggml-decoder.cpp:1641), but its graph input keeps -1 (:708). process_view_input_new (openvino/utils.cpp:316-336) has no match for expected-static/actual-dynamic, so these views fall through to process_single_view. For decode (n_outputs = 1), the last-layer ffn_moe_weighted view (llama-graph.cpp:2273) becomes Slice(axis=2, i, i+1) on an axis of size 1, which is empty for every expert after the first (openvino/utils.cpp:631-655).

qwen3moe.cpp:123-125 gathers inp_out_ids inside the last layer, before the MoE FFN, and build_inp_out_ids always builds the input. This therefore runs on every decode step, because fixed_token_count == 1 enables fixed mode. gpt-oss gathers after all layers by default and is unaffected.

Impact: Qwen3-MoE MXFP4 (and openai-moe with embeddings_nextn_masked) on the OpenVINO CPU device either loses all but one expert from the last-layer MoE sum or fails on the output shape. Traced from the code; not reproduced on a device.

Suggested fix: keep out_ids-derived dims dynamic, as on master. cgraph nodes are topologically ordered, so one pass right after compute_node_dynamic_dims(); in the graph constructor collects every descendant of an out_ids gather:

// ggml-decoder.h, next to m_node_dynamic_dims
std::set<const ggml_tensor *> m_out_ids_derived;

// ggml-decoder.cpp, graph constructor, after compute_node_dynamic_dims();
if (m_fixed_token_shape) {
    for (int i = 0; i < m_cgraph->n_nodes; ++i) {
        const ggml_tensor * node = m_cgraph->nodes[i];
        for (int j = 0; j < GGML_MAX_SRC; ++j) {
            const ggml_tensor * src = node->src[j];
            if (src != nullptr && (is_output_idx(src, node) || m_out_ids_derived.count(src))) {
                m_out_ids_derived.insert(node);
                break;
            }
        }
    }
}

Then change the fixed branch of both view getters to m_fixed_token_shape && !m_out_ids_derived.count(tensor) (src_tensor at :1463).

Every tensor this touches returns to master's -1, and nothing changes outside fixed mode. It also removes the n_outputs dependence of the compiled views, which graph_key does not cover. Not compiled against OpenVINO.

@yingying0906 yingying0906 Oct 7, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, confirmed in the code. out_ids stays -1 but the views became static in fixed mode, so they fall through to re-slicing. Applied your fix in 42957c3. gpt-oss gathers after all layers so my VM runs did not hit it, and I don't have a Qwen3-MoE MXFP4 model to reproduce, so I checked it with gpt-oss regression runs only: 91 MXFP4 op cases, 32-token prefill, 1-token decode and ubatch=16 all pass.

}

return shape;
Expand All @@ -1452,7 +1459,9 @@ ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx,
if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) {
int dynamic_dim_index = dynamic_it->second;
// GGML uses reverse indexing, so convert to OpenVINO indexing
shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1;
shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() :
m_fixed_token_shape ? src_tensor->ne[dynamic_dim_index] :
-1;
}

return shape;
Expand Down
4 changes: 3 additions & 1 deletion ggml/src/ggml-openvino/ggml-decoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
bool is_stateful = false,
bool model_is_splitted = false,
bool is_prefill = false,
int prefill_chunk_size = 256);
int prefill_chunk_size = 256,
bool fixed_token_shape = false);

// Naive graph decoder
GgmlOvDecoder(ggml_cgraph * cgraph, std::map<std::string, std::shared_ptr<ov::Node>> & model_weights);
Expand Down Expand Up @@ -301,6 +302,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
void set_compute_params(const ComputeParams & compute_params) { m_compute_params = compute_params; }

bool m_is_static = false;
bool m_fixed_token_shape = false;
bool m_is_stateful = false;
bool m_is_prefill = false;
bool m_naive = false;
Expand Down
8 changes: 7 additions & 1 deletion ggml/src/ggml-openvino/model-cache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
bool fa,
const int32_t * rope_params,
int rope_len,
uint64_t extra_cfg) {
uint64_t extra_cfg,
int64_t fixed_token_count) {
uint64_t h = FNV_OFFSET;

// Topology: node count + each node's op and name (cheap, and distinguishes
Expand All @@ -200,6 +201,11 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
h = fnv1a(h, rope_params, sizeof(int32_t) * static_cast<size_t>(rope_len));
}
h = fnv1a_u64(h, extra_cfg);
if (fixed_token_count > 0) {
constexpr char token_shape_tag[] = "fixed-token-shape";
h = fnv1a(h, token_shape_tag, sizeof(token_shape_tag) - 1);
h = fnv1a_u64(h, static_cast<uint64_t>(fixed_token_count));
}
const std::string ver = ov_version_string();
h = fnv1a(h, ver.data(), ver.size());

Expand Down
3 changes: 2 additions & 1 deletion ggml/src/ggml-openvino/model-cache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
bool fa,
const int32_t * rope_params,
int rope_len,
uint64_t extra_cfg);
uint64_t extra_cfg,
int64_t fixed_token_count = 0);

// Path to the compiled-blob file for a fingerprint (<dir>/<hex>.blob).
std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint);
Expand Down
59 changes: 51 additions & 8 deletions ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "gather_matmul.hpp"
#include "ggml-openvino/ggml-openvino-extra.h"

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <limits>
Expand Down Expand Up @@ -117,6 +118,32 @@ ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & conte
const int64_t k_blocks = static_cast<int64_t>(packed_shape[3]);
const int64_t qk = 32;
const int64_t cols = k_blocks * qk;
const bool is_cpu = ggml_openvino_get_device_name() == "CPU";

if (is_cpu) {
const ov::PartialShape id_shape = ids.get_partial_shape();
const ov::PartialShape act_shape = activations.get_partial_shape();
if (id_shape.rank().is_static() && id_shape.rank().get_length() == 4 && act_shape.rank().is_static() &&
act_shape.rank().get_length() == 4 && id_shape[3].is_static() && id_shape[3].get_length() != 0 &&
rows != 0 && cols != 0) {
// The CPU JIT uses signed 32-bit gather offsets for the decoded weights.
static constexpr size_t max_selected_bytes = 512ULL << 20;
const size_t max_chunk_tokens =
std::max<size_t>(1, max_selected_bytes / sizeof(float) / id_shape[3].get_length() / rows / cols);
if (id_shape[2].is_static() && act_shape[1].is_static() && id_shape[2].get_length() > max_chunk_tokens) {
const size_t n_tokens = id_shape[2].get_length();
ov::OutputVector chunks;
chunks.reserve((n_tokens + max_chunk_tokens - 1) / max_chunk_tokens);
for (size_t begin = 0; begin < n_tokens; begin += max_chunk_tokens) {
const size_t end = std::min(n_tokens, begin + max_chunk_tokens);
chunks.push_back(translate_mul_mat_id_mxfp4_packed(context, expert_weights,
slice_axis(activations, 1, begin, end),
slice_axis(ids, 2, begin, end)));
}
return std::make_shared<ov::op::v0::Concat>(chunks, 1);
}
}
}

auto packed_shape_4d = const_i64({n_expert, rows, k_blocks, 17});
expert_weights = std::make_shared<ov::op::v1::Reshape>(expert_weights, packed_shape_4d, false);
Expand Down Expand Up @@ -144,19 +171,35 @@ ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & conte
e8m0_lut[0] = std::numeric_limits<float>::min() / 2.0f;
e8m0_lut[255] = std::numeric_limits<float>::quiet_NaN();

auto f4_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{f4e2m1_lut.size()}, f4e2m1_lut);
auto scale_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{e8m0_lut.size()}, e8m0_lut);

auto selected_packed_weights = std::make_shared<ov::op::v8::Gather>(expert_weights, ids, gather_axis);
auto scale_byte = slice_axis(selected_packed_weights, 4, 0, 1);
auto qs = slice_axis(selected_packed_weights, 4, 1, 17);
auto low = std::make_shared<ov::op::v13::BitwiseAnd>(
qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {0x0F}), ov::op::AutoBroadcastType::NUMPY);
auto high_shift = std::make_shared<ov::op::v15::BitwiseRightShift>(
qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {4}), ov::op::AutoBroadcastType::NUMPY);
auto nibbles = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{low, high_shift}, 4);
auto nibble_indices = std::make_shared<ov::op::v0::Convert>(nibbles, ov::element::i32);
auto weights_f32 = std::make_shared<ov::op::v8::Gather>(f4_lut, nibble_indices, gather_axis);
ov::Output<ov::Node> weights_f32;
if (is_cpu) {
std::vector<float> low_lut(256);
std::vector<float> high_lut(256);
for (size_t i = 0; i < low_lut.size(); ++i) {
low_lut[i] = f4e2m1_lut[i & 0x0F];
high_lut[i] = f4e2m1_lut[i >> 4];
}
auto low_lut_node = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{low_lut.size()}, low_lut);
auto high_lut_node = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{high_lut.size()}, high_lut);
auto byte_indices = std::make_shared<ov::op::v0::Convert>(qs, ov::element::i32);
auto low = std::make_shared<ov::op::v8::Gather>(low_lut_node, byte_indices, gather_axis);
auto high = std::make_shared<ov::op::v8::Gather>(high_lut_node, byte_indices, gather_axis);
weights_f32 = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{low, high}, 4);
} else {
auto f4_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{f4e2m1_lut.size()}, f4e2m1_lut);
auto low = std::make_shared<ov::op::v13::BitwiseAnd>(
qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {0x0F}), ov::op::AutoBroadcastType::NUMPY);
auto high = std::make_shared<ov::op::v15::BitwiseRightShift>(
qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {4}), ov::op::AutoBroadcastType::NUMPY);
auto nibbles = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{low, high}, 4);
auto nibble_indices = std::make_shared<ov::op::v0::Convert>(nibbles, ov::element::i32);
weights_f32 = std::make_shared<ov::op::v8::Gather>(f4_lut, nibble_indices, gather_axis);
}

auto scale_indices = std::make_shared<ov::op::v0::Convert>(scale_byte, ov::element::i32);
auto scales_f32 = std::make_shared<ov::op::v8::Gather>(scale_lut, scale_indices, gather_axis);
Expand Down
Loading
Loading