fix: improve quantization and runtime correctness

gptqmodel/looper/loop_processor.py

@@ -51,6 +51,13 @@
 ]
 
 
+def _format_gib(value: float) -> str:
+    """Formats a GiB value without unnecessary trailing zeros."""
+
+    text = f"{value:.2f}".rstrip("0").rstrip(".")
+    return f"{text}G"
+
+
 class _ThreadSafeDict(dict):
     """Dictionary with synchronized mutations and snapshot-based iteration."""
 
@@ -687,16 +694,6 @@ def device_memory_report(self) -> str:
         if not snapshot:
             return "n/a"
 
-        def _format_gib(value: float) -> str:
-            """Formats a GiB value without unnecessary trailing zeros."""
-
-            text = f"{value:.2f}"
-            if text.endswith("00"):
-                text = text[:-2]
-            elif text.endswith("0"):
-                text = text[:-1]
-            return f"{text}G"
-
         grouped: Dict[str, List[Tuple[str, float, int]]] = {}
         for order, (device_id, value) in enumerate(snapshot.items()):
             family, _, index = device_id.partition(":")
@@ -945,12 +942,12 @@ def get_max_memory() -> str:
 
     stats_0 = torch.cuda.memory_stats(DEVICE_0)
     active_0 = stats_0.get("active_bytes.all.current", 0) / 1024 ** 2
-    peak_active_0 = stats_0.get("active_bytes.all.peak", 0) / 1024 ** 2
+    stats_0.get("active_bytes.all.peak", 0) / 1024 ** 2
 
     if torch.cuda.device_count() > 1:
         stats_1 = torch.cuda.memory_stats(DEVICE_1)
         active_1 = stats_1.get("active_bytes.all.current", 0) / 1024 ** 2
-        peak_active_1 = stats_1.get("active_bytes.all.peak", 0) / 1024 ** 2
+        stats_1.get("active_bytes.all.peak", 0) / 1024 ** 2
 
         max_memory = f"{active_0:.2f}MB, {active_1:.2f}MB"
     else:

gptqmodel/models/definitions/ernie4_5_moe.py

@@ -35,7 +35,7 @@ class Ernie4_5_MoeQModel(BaseQModel):
                     "down_proj": ("down_proj:1",),
                 },
                 "experts": {
-                    "#": ("gate_proj:0", "upe_proj:0", "down_proj:1"),
+                    "#": ("gate_proj:0", "up_proj:0", "down_proj:1"),
                 },
             },
         }

gptqmodel/models/definitions/ernie4_5_vl_moe.py

@@ -53,13 +53,13 @@ class Ernie4_5_VLMoeQModel(BaseQModel):
                 "text_moe": {
                     "gate": ("gate:!",),
                     "experts": {
-                        "#": ("gate_proj:0", "upe_proj:0", "down_proj:1"),
+                        "#": ("gate_proj:0", "up_proj:0", "down_proj:1"),
                     },
                 },
                 "vision_moe": {
                     "gate": ("gate:!",),
                     "experts": {
-                        "#": ("gate_proj:0", "upe_proj:0", "down_proj:1"),
+                        "#": ("gate_proj:0", "up_proj:0", "down_proj:1"),
                     },
                 }
             },

gptqmodel/models/writer.py

@@ -1143,7 +1143,7 @@ def get_model_with_quantize(self, qcfg, model_id_or_path):
         prepare_remote_code_compat(config)
 
         with suspend_hf_weight_init():
-            model = cls.loader.from_config(
+            model = self.loader.from_config(
                 config, dtype=torch.float16
             )
 
@@ -1173,7 +1173,7 @@ def get_model_with_quantize(self, qcfg, model_id_or_path):
                 qcfg=qcfg,
                 quant_result=modules,
                 backend=BACKEND.AUTO,
-                lm_head_name=cls.lm_head,
+                lm_head_name=self.lm_head,
                 pack=True,
                 device=DEVICE.CPU,
             )

gptqmodel/quantization/gptq.py

@@ -11,7 +11,6 @@
 import time
 from typing import Dict, Optional, Tuple
 
-import numpy as np
 import torch
 import torch.nn as nn
 import transformers
@@ -161,8 +160,8 @@ def get_number_of_rows_and_cols(layer: nn.Module):
         # transformers.Conv1D: weight shape is (n_in, n_out)
         return layer.weight.shape[1], layer.weight.shape[0]
     else:
-        # weight shape is (n_out, n_in)
-        return layer.weight.shape[0], np.prod(layer.weight.shape[1:])
+        # weight shape is (n_out, n_in); math.prod keeps `columns` a plain int
+        return layer.weight.shape[0], math.prod(layer.weight.shape[1:])
 
 
 class GPTQ:
@@ -684,8 +683,17 @@ def create_H(self, target_device):
         return torch.zeros((self.columns, self.columns), dtype=torch.float32,
                            device=self._select_hessian_target_device(target_device))
 
-    def _fallback_quantize(self, strategy: FallbackStrategy, blocksize: int):
-        """Apply a lightweight quantization fallback using the requested strategy."""
+    def _fallback_quantize(
+        self,
+        strategy: FallbackStrategy,
+        blocksize: int,
+        target_device: Optional[torch.device] = None,
+    ):
+        """Apply a lightweight quantization fallback using the requested strategy.
+
+        ``target_device`` is the device the weight clone is quantized on; when
+        None it is taken from ``self.H`` if present, else the weight.
+        """
         maxq = 2 ** self.qcfg.bits - 1
         sigma = 3.0
         effective_group_size = self.qcfg.group_size if self.qcfg.group_size != -1 else self.columns
@@ -697,7 +705,8 @@ def _fallback_quantize(self, strategy: FallbackStrategy, blocksize: int):
             mse_steps = smooth_method.steps
             mse_maxshrink = smooth_method.maxshrink
 
-        target_device = self.H.device if self.H is not None else self.module.weight.device
+        if target_device is None:
+            target_device = self.H.device if self.H is not None else self.module.weight.device
         W = self.clone_module(device=target_device)
         Q = torch.empty_like(W)
         scale_chunks = []
@@ -989,9 +998,19 @@ def quantize(
                 f"Quantization: Module `{self.name}` -> "
                 f"Using `{resolved_strategy.value}` fallback quantization (observed {self.nsamples} samples, threshold={threshold_text}{threshold_info}, max_total={self.expected_nsamples})."
             )
-            self.H = self.create_H(target_device=target_device)
+            # The fallback never reads the Hessian: release the fp32 XtX
+            # partials (columns^2 x 4 B each) instead of folding them into a
+            # throwaway zero H. Resolve the compute device first, since the
+            # clear changes what _select_hessian_target_device returns.
+            with self.lock:
+                fallback_device = self._select_hessian_target_device(target_device)
+                self._device_hessian_partials.clear()
+                self._device_sample_counts.clear()
+                self._hessian_dirty = False
 
-            return self._fallback_quantize(resolved_strategy, blocksize)
+            return self._fallback_quantize(
+                resolved_strategy, blocksize, target_device=fallback_device
+            )
         else:
             use_hessian = True
             self.finalize_hessian(target_device=target_device)
@@ -1446,6 +1465,13 @@ def reset_workspace_stats(self) -> None:
         self._borrow_workspace_last_chunk_rows = None
 
     def free(self):
+        # The task object outlives free() in processor.tasks until layer
+        # end, so no path may leave a Hessian partial behind here.
+        with self.lock:
+            self._device_hessian_partials.clear()
+            self._device_sample_counts.clear()
+            self._hessian_dirty = False
+
         if hasattr(self, "H"):
             del self.H
         del self.quantizer

gptqmodel/utils/importer.py

@@ -554,7 +554,9 @@ def select_quant_linear(
                 if os.environ.get("DEBUG"):
                     log.info(f"skip {k} for unsupported device `{device}`")
                 continue
-            supports_sharded_load = getattr(cls, "SUPPORTS_SHARDED_LOAD", cls.SUPPORTS_SHARDS)
+            supports_sharded_load = getattr(
+                cls, "SUPPORTS_SHARDED_LOAD", getattr(cls, "SUPPORTS_SHARDS", True)
+            )
             if is_sharded and not supports_sharded_load:
                 if os.environ.get("DEBUG"):
                     log.info(f"skip {k} because sharded checkpoints are not supported")
@@ -623,7 +625,9 @@ def select_quant_linear(
     # Handle the case where backend is not AUTO.
     qlinear = get_kernel_for_backend(backend, quant_method, format)
 
-    supports_sharded_load = getattr(qlinear, "SUPPORTS_SHARDED_LOAD", qlinear.SUPPORTS_SHARDS)
+    supports_sharded_load = getattr(
+        qlinear, "SUPPORTS_SHARDED_LOAD", getattr(qlinear, "SUPPORTS_SHARDS", True)
+    )
     if is_sharded and not supports_sharded_load:
         raise ValueError(f"Selected backend `{backend}` with kernel `{qlinear.__name__}` does not support sharded checkpoints.")
 

tests/kernels/test_selection.py

@@ -614,3 +614,36 @@ def test_select_quant_linear_single_select_keeps_model_wide_kernel(monkeypatch):
     # Marlin/Exllama/Triton do not support base bits=3, so the model-wide kernel
     # must be TorchLinear, which supports both the base 3-bit and dynamic 4-bit layers.
     assert selected is TorchLinear
+
+
+def test_sharded_select_tolerates_kernel_without_shard_attrs(monkeypatch):
+    class BareKernel:
+        # Defines neither SUPPORTS_SHARDED_LOAD nor SUPPORTS_SHARDS; the
+        # sharded-load guard must not evaluate a missing attribute eagerly
+        # and treats absent declarations as shard-capable.
+        SUPPORTS_DEVICES = [DEVICE.ALL]
+
+        @classmethod
+        def validate(cls, **_):
+            return True, None
+
+    monkeypatch.setitem(
+        AUTO_BACKEND_KERNEL_MAPPING[METHOD.QQQ],
+        FORMAT.QQQ,
+        OrderedDict([(BACKEND.QQQ_TORCH, BareKernel)]),
+    )
+
+    qlinear_cls = select_quant_linear(
+        bits=4,
+        group_size=128,
+        desc_act=False,
+        sym=True,
+        device=torch.device("cpu"),
+        backend=BACKEND.AUTO,
+        format=FORMAT.QQQ,
+        quant_method=METHOD.QQQ,
+        pack_dtype=torch.int32,
+        is_sharded=True,
+    )
+
+    assert qlinear_cls is BareKernel

tests/module_tree/test_ernie_expert_projections.py

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: 2026 ModelCloud.ai
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from gptqmodel.models.definitions.ernie4_5_moe import Ernie4_5_MoeQModel
+from gptqmodel.models.definitions.ernie4_5_vl_moe import Ernie4_5_VLMoeQModel
+
+
+def _collect_strings(node):
+    if isinstance(node, str):
+        yield node
+    elif isinstance(node, dict):
+        for key, value in node.items():
+            yield from _collect_strings(key)
+            yield from _collect_strings(value)
+    elif isinstance(node, (list, tuple)):
+        for item in node:
+            yield from _collect_strings(item)
+
+
+@pytest.mark.parametrize("model_cls", [Ernie4_5_MoeQModel, Ernie4_5_VLMoeQModel])
+def test_expert_projections_use_up_proj(model_cls):
+    tokens = {entry.split(":", 1)[0] for entry in _collect_strings(model_cls.module_tree)}
+    # Regression: routed-expert entries carried a "upe_proj" typo while the
+    # shared-expert entries correctly used "up_proj" (ERNIE 4.5 MoE modeling
+    # names the expert projections gate_proj/up_proj/down_proj).
+    assert "upe_proj" not in tokens
+    assert "up_proj" in tokens