Fix MiniMax H3 GGUF model and encoder loading

loader.py

@@ -9,7 +9,7 @@
 from .ops import GGMLTensor
 from .dequant import is_quantized, dequantize_tensor
 
-IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image"}
+IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image", "minimax_h3"}
 TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"}
 VIS_TYPE_LIST = {"clip-vision", "mmproj"}
 
@@ -219,6 +219,21 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F
     "ln2.": "norm2.",
 }
 
+QWEN3_VISION_SD_MAP = {
+    "mm.0.": "visual.merger.linear_fc1.",
+    "mm.2.": "visual.merger.linear_fc2.",
+    "v.post_ln.": "visual.merger.norm.",
+    "v.patch_embd": "visual.patch_embed.proj",
+    "v.position_embd.": "visual.pos_embed.",
+    "v.blk.": "visual.blocks.",
+    "ffn_up": "mlp.linear_fc1",
+    "ffn_down": "mlp.linear_fc2",
+    "attn_qkv.": "attn.qkv.",
+    "attn_out.": "attn.proj.",
+    "ln1.": "norm1.",
+    "ln2.": "norm2.",
+}
+
 def sd_map_replace(raw_sd, key_map):
     sd = {}
     for k,v in raw_sd.items():
@@ -290,7 +305,7 @@ def gguf_mmproj_loader(path):
             target.append(fname)
 
     if len(target) == 0:
-        logging.error(f"Error: Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}')! Qwen-Image-Edit will be broken!")
+        logging.warning(f"No matching mmproj found for '{tenc_fname}' (matching: '{tenc}').")
         return {}
     if len(target) > 1:
         logging.error(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.")
@@ -305,8 +320,16 @@ def gguf_mmproj_loader(path):
         w2 = dequantize_tensor(vsd.pop("v.patch_embd.weight.1"), dtype=torch.float32)
         vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2)
 
-    # run main replacement
-    vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP)
+    # DeepStack tensors use vision block indices, not merger indices.
+    deepstack_blocks = sorted({int(k.split(".")[2]) for k in vsd if k.startswith("v.deepstack.")})
+    if deepstack_blocks:
+        key_map = QWEN3_VISION_SD_MAP.copy()
+        for i, block in enumerate(deepstack_blocks):
+            for src, dst in (("norm", "norm"), ("fc1", "linear_fc1"), ("fc2", "linear_fc2")):
+                key_map[f"v.deepstack.{block}.{src}."] = f"visual.deepstack_merger_list.{i}.{dst}."
+        vsd = sd_map_replace(vsd, key_map)
+    else:
+        vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP)
 
     # handle split Q/K/V
     if "visual.blocks.0.attn_q.weight" in vsd:
@@ -498,8 +521,17 @@ def gguf_clip_loader(path):
             sd = sd_map_replace(sd, LLAMA_SD_MAP)
         if arch == "llama":
             sd = llama_permute(sd, 32, 8) # L3 / Mistral
-        if arch == "qwen2vl":
+        if arch in {"qwen2vl", "qwen3vl"}:
             vsd = gguf_mmproj_loader(path)
+            if arch == "qwen3vl":
+                norm = sd.get("model.layers.0.post_attention_layernorm.weight")
+                if norm is not None and norm.shape[0] == 5120:
+                    if not vsd:
+                        raise ValueError(f"Missing Qwen3-VL mmproj for {os.path.basename(path)!r}; place the matching mmproj GGUF beside the text encoder.")
+                elif vsd:
+                    # ComfyUI detects the smaller Qwen3-VL encoders in HF layout.
+                    sd = {k.replace("model.", "model.language_model.", 1) if k.startswith("model.") else k: v for k, v in sd.items()}
+                    vsd = {"model." + k: v for k, v in vsd.items()}
             sd.update(vsd)
     else:
         pass

tests/test_minimax_loading.py

@@ -0,0 +1,139 @@
+"""Run with PYTHONPATH=/path/to/ComfyUI python -m pytest -q tests."""
+
+import importlib
+import importlib.util
+from pathlib import Path
+import sys
+
+import gguf
+import pytest
+import torch
+
+from comfy.cli_args import args
+
+args.cpu = True
+spec = importlib.util.spec_from_file_location("comfyui_gguf", Path(__file__).resolve().parents[1] / "__init__.py")
+package = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = package
+spec.loader.exec_module(package)
+loader = importlib.import_module("comfyui_gguf.loader")
+convert = importlib.import_module("comfyui_gguf.tools.convert")
+
+
+def test_minimax_architecture():
+    keys = {"video_patch_proj.weight", "audio_patch_proj.weight", "final_layer.video_out.weight", "final_layer.audio_out.weight"}
+    assert convert.detect_arch(keys).arch == "minimax_h3"
+    assert "minimax_h3" in loader.IMG_ARCH_LIST
+    assert not convert.is_model_arch(convert.ModelMiniMaxH3, keys - {"audio_patch_proj.weight"})
+
+
+def test_qwen3_vision_mapping(tmp_path, monkeypatch):
+    encoder = tmp_path / "Qwen3VL-32B-Thinking-Q4_K_M.gguf"
+    projector = tmp_path / "Qwen3VL-32B-Thinking-mmproj-Q8_0.gguf"
+    projector.touch()
+    (tmp_path / "Qwen3VL-8B-Instruct-mmproj-Q8_0.gguf").touch()
+    first = torch.ones(2, 3, 4, 4)
+    second = first * 2
+    weight = torch.randn(2, 2)
+    sd = {
+        "v.patch_embd.weight": first,
+        "v.patch_embd.weight.1": second,
+        "v.patch_embd.bias": weight,
+        "v.position_embd.weight": weight,
+        "v.blk.0.attn_qkv.weight": weight,
+        "v.blk.0.attn_out.weight": weight,
+        "v.blk.0.ffn_up.weight": weight,
+        "v.blk.0.ffn_down.weight": weight,
+        "v.blk.0.ln1.weight": weight,
+        "v.blk.0.ln2.weight": weight,
+        "v.post_ln.weight": weight,
+        "mm.0.weight": weight,
+        "mm.2.weight": weight,
+    }
+    for block in (24, 8, 16):
+        for name in ("norm", "fc1", "fc2"):
+            sd[f"v.deepstack.{block}.{name}.weight"] = torch.full((2, 2), float(block))
+
+    def load(path, is_text_model):
+        assert path == str(projector)
+        assert is_text_model
+        return sd, {}
+
+    monkeypatch.setattr(loader, "gguf_sd_loader", load)
+    result = loader.gguf_mmproj_loader(str(encoder))
+    patch = result.pop("visual.patch_embed.proj.weight")
+    torch.testing.assert_close(patch, torch.stack((first, second), dim=2))
+    expected = {
+        "visual.patch_embed.proj.bias", "visual.pos_embed.weight",
+        "visual.blocks.0.attn.qkv.weight", "visual.blocks.0.attn.proj.weight",
+        "visual.blocks.0.mlp.linear_fc1.weight", "visual.blocks.0.mlp.linear_fc2.weight",
+        "visual.blocks.0.norm1.weight", "visual.blocks.0.norm2.weight",
+        "visual.merger.norm.weight", "visual.merger.linear_fc1.weight", "visual.merger.linear_fc2.weight",
+    }
+    expected.update(f"visual.deepstack_merger_list.{i}.{name}.weight" for i in range(3) for name in ("norm", "linear_fc1", "linear_fc2"))
+    assert set(result) == expected
+    for name, value in result.items():
+        if not name.startswith("visual.deepstack_merger_list."):
+            assert value is weight
+    for i, block in enumerate((8, 16, 24)):
+        for name in ("norm", "linear_fc1", "linear_fc2"):
+            torch.testing.assert_close(result[f"visual.deepstack_merger_list.{i}.{name}.weight"], torch.full((2, 2), float(block)))
+
+
+@pytest.mark.parametrize("width,prefix", [(5120, ""), (4096, "model."), (2560, "model.")])
+def test_qwen3_clip_includes_vision(monkeypatch, width, prefix):
+    sd = {"blk.0.ffn_norm.weight": torch.empty(width)}
+    vision = {"visual.deepstack_merger_list.0.norm.weight": torch.empty(4)}
+    monkeypatch.setattr(loader, "gguf_sd_loader", lambda *a, **k: (sd, {"arch_str": "qwen3vl"}))
+    monkeypatch.setattr(loader, "gguf_mmproj_loader", lambda path: vision)
+    result = loader.gguf_clip_loader("qwen3vl.gguf")
+    assert prefix + "visual.deepstack_merger_list.0.norm.weight" in result
+    language_prefix = "model.language_model." if prefix else "model."
+    assert language_prefix + "layers.0.post_attention_layernorm.weight" in result
+
+
+def test_qwen3_missing_projector_is_an_error(monkeypatch):
+    monkeypatch.setattr(loader, "gguf_sd_loader", lambda *a, **k: ({"blk.0.ffn_norm.weight": torch.empty(5120)}, {"arch_str": "qwen3vl"}))
+    monkeypatch.setattr(loader, "gguf_mmproj_loader", lambda path: {})
+    with pytest.raises(ValueError, match="Missing Qwen3-VL mmproj"):
+        loader.gguf_clip_loader("qwen3vl.gguf")
+
+
+def test_qwen2_vision_mapping_is_preserved(tmp_path, monkeypatch):
+    (tmp_path / "Qwen2VL-mmproj.gguf").touch()
+    weight = torch.randn(2, 2)
+    sd = {"mm.0.weight": weight, "v.post_ln.weight": weight, "v.blk.0.ffn_up.weight": weight}
+    monkeypatch.setattr(loader, "gguf_sd_loader", lambda *a, **k: (sd, {}))
+    result = loader.gguf_mmproj_loader(str(tmp_path / "Qwen2VL-Q4_K_M.gguf"))
+    assert set(result) == {"visual.merger.mlp.0.weight", "visual.merger.ln_q.weight", "visual.blocks.0.mlp.up_proj.weight"}
+
+
+@pytest.mark.parametrize("width", [2560, 4096])
+def test_smaller_qwen3_text_only_loading_is_preserved(monkeypatch, width):
+    sd = {"blk.0.ffn_norm.weight": torch.empty(width)}
+    monkeypatch.setattr(loader, "gguf_sd_loader", lambda *a, **k: (sd, {"arch_str": "qwen3vl"}))
+    monkeypatch.setattr(loader, "gguf_mmproj_loader", lambda path: {})
+    result = loader.gguf_clip_loader("qwen3vl.gguf")
+    assert set(result) == {"model.layers.0.post_attention_layernorm.weight"}
+
+
+@pytest.mark.parametrize("arch", [None, "minimax_h3"])
+def test_minimax_gguf_loading(tmp_path, arch):
+    path = tmp_path / "minimax.gguf"
+    writer = gguf.GGUFWriter(path, arch="minimax_h3")
+    if arch is None:
+        writer.kv_data[0].pop("general.architecture")
+    keys = ("video_patch_proj.weight", "audio_patch_proj.weight", "final_layer.video_out.weight", "final_layer.audio_out.weight")
+    weight = torch.arange(4, dtype=torch.float32).reshape(2, 2)
+    for key in keys:
+        writer.add_tensor(key, weight.numpy())
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file()
+    writer.close()
+
+    sd, extra = loader.gguf_sd_loader(path)
+    assert extra["arch_str"] == "minimax_h3"
+    assert set(sd) == set(keys)
+    for value in sd.values():
+        torch.testing.assert_close(value.as_subclass(torch.Tensor), weight)

tools/convert.py

@@ -145,8 +145,15 @@ class ModelLumina2(ModelTemplate):
         ("cap_embedder.1.weight", "context_refiner.0.attention.qkv.weight")
     ]
 
+class ModelMiniMaxH3(ModelTemplate):
+    arch = "minimax_h3"
+    keys_detect = [
+        ("video_patch_proj.weight", "audio_patch_proj.weight", "final_layer.video_out.weight", "final_layer.audio_out.weight"),
+    ]
+    keys_hiprec = ["adaln_t_table", "rope.inv_freq"]
+
 arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, CosmosPredict2, 
-             ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2]
+             ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2, ModelMiniMaxH3]
 
 def is_model_arch(model, state_dict):
     # check if model is correct