From a00548de9131ab24b72a8297374ecc5184ddbc7f Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Fri, 4 Sep 2026 15:28:40 +0800 Subject: [PATCH] fix: port upstream GPTQModel bug fixes --- gptqmodel/looper/loop_processor.py | 21 +-- gptqmodel/models/definitions/ernie4_5_moe.py | 2 +- .../models/definitions/ernie4_5_vl_moe.py | 4 +- gptqmodel/models/writer.py | 4 +- gptqmodel/quantization/gptq.py | 42 ++++- gptqmodel/utils/importer.py | 8 +- tests/kernels/test_selection.py | 33 ++++ .../test_ernie_expert_projections.py | 29 +++ tests/test_fallback_hessian_release.py | 172 ++++++++++++++++++ tests/test_format_gib.py | 23 +++ tests/test_writer_instance_attrs.py | 17 ++ 11 files changed, 328 insertions(+), 27 deletions(-) create mode 100644 tests/module_tree/test_ernie_expert_projections.py create mode 100644 tests/test_fallback_hessian_release.py create mode 100644 tests/test_format_gib.py create mode 100644 tests/test_writer_instance_attrs.py diff --git a/gptqmodel/looper/loop_processor.py b/gptqmodel/looper/loop_processor.py index d8de58e32..73d31a5b7 100644 --- a/gptqmodel/looper/loop_processor.py +++ b/gptqmodel/looper/loop_processor.py @@ -51,6 +51,13 @@ ] +def _format_gib(value: float) -> str: + """Formats a GiB value without unnecessary trailing zeros.""" + + text = f"{value:.2f}".rstrip("0").rstrip(".") + return f"{text}G" + + class _ThreadSafeDict(dict): """Dictionary with synchronized mutations and snapshot-based iteration.""" @@ -687,16 +694,6 @@ def device_memory_report(self) -> str: if not snapshot: return "n/a" - def _format_gib(value: float) -> str: - """Formats a GiB value without unnecessary trailing zeros.""" - - text = f"{value:.2f}" - if text.endswith("00"): - text = text[:-2] - elif text.endswith("0"): - text = text[:-1] - return f"{text}G" - grouped: Dict[str, List[Tuple[str, float, int]]] = {} for order, (device_id, value) in enumerate(snapshot.items()): family, _, index = device_id.partition(":") @@ -945,12 +942,12 @@ def get_max_memory() -> str: stats_0 = torch.cuda.memory_stats(DEVICE_0) active_0 = stats_0.get("active_bytes.all.current", 0) / 1024 ** 2 - peak_active_0 = stats_0.get("active_bytes.all.peak", 0) / 1024 ** 2 + stats_0.get("active_bytes.all.peak", 0) / 1024 ** 2 if torch.cuda.device_count() > 1: stats_1 = torch.cuda.memory_stats(DEVICE_1) active_1 = stats_1.get("active_bytes.all.current", 0) / 1024 ** 2 - peak_active_1 = stats_1.get("active_bytes.all.peak", 0) / 1024 ** 2 + stats_1.get("active_bytes.all.peak", 0) / 1024 ** 2 max_memory = f"{active_0:.2f}MB, {active_1:.2f}MB" else: diff --git a/gptqmodel/models/definitions/ernie4_5_moe.py b/gptqmodel/models/definitions/ernie4_5_moe.py index 92cae6694..b4ff9e6d3 100644 --- a/gptqmodel/models/definitions/ernie4_5_moe.py +++ b/gptqmodel/models/definitions/ernie4_5_moe.py @@ -35,7 +35,7 @@ class Ernie4_5_MoeQModel(BaseQModel): "down_proj": ("down_proj:1",), }, "experts": { - "#": ("gate_proj:0", "upe_proj:0", "down_proj:1"), + "#": ("gate_proj:0", "up_proj:0", "down_proj:1"), }, }, } diff --git a/gptqmodel/models/definitions/ernie4_5_vl_moe.py b/gptqmodel/models/definitions/ernie4_5_vl_moe.py index 4a9dd8a4c..12f20a865 100644 --- a/gptqmodel/models/definitions/ernie4_5_vl_moe.py +++ b/gptqmodel/models/definitions/ernie4_5_vl_moe.py @@ -53,13 +53,13 @@ class Ernie4_5_VLMoeQModel(BaseQModel): "text_moe": { "gate": ("gate:!",), "experts": { - "#": ("gate_proj:0", "upe_proj:0", "down_proj:1"), + "#": ("gate_proj:0", "up_proj:0", "down_proj:1"), }, }, "vision_moe": { "gate": ("gate:!",), "experts": { - "#": ("gate_proj:0", "upe_proj:0", "down_proj:1"), + "#": ("gate_proj:0", "up_proj:0", "down_proj:1"), }, } }, diff --git a/gptqmodel/models/writer.py b/gptqmodel/models/writer.py index 6981ced18..c93429bea 100644 --- a/gptqmodel/models/writer.py +++ b/gptqmodel/models/writer.py @@ -1143,7 +1143,7 @@ def get_model_with_quantize(self, qcfg, model_id_or_path): prepare_remote_code_compat(config) with suspend_hf_weight_init(): - model = cls.loader.from_config( + model = self.loader.from_config( config, dtype=torch.float16 ) @@ -1173,7 +1173,7 @@ def get_model_with_quantize(self, qcfg, model_id_or_path): qcfg=qcfg, quant_result=modules, backend=BACKEND.AUTO, - lm_head_name=cls.lm_head, + lm_head_name=self.lm_head, pack=True, device=DEVICE.CPU, ) diff --git a/gptqmodel/quantization/gptq.py b/gptqmodel/quantization/gptq.py index 78ed6c43c..c83693d0c 100644 --- a/gptqmodel/quantization/gptq.py +++ b/gptqmodel/quantization/gptq.py @@ -11,7 +11,6 @@ import time from typing import Dict, Optional, Tuple -import numpy as np import torch import torch.nn as nn import transformers @@ -161,8 +160,8 @@ def get_number_of_rows_and_cols(layer: nn.Module): # transformers.Conv1D: weight shape is (n_in, n_out) return layer.weight.shape[1], layer.weight.shape[0] else: - # weight shape is (n_out, n_in) - return layer.weight.shape[0], np.prod(layer.weight.shape[1:]) + # weight shape is (n_out, n_in); math.prod keeps `columns` a plain int + return layer.weight.shape[0], math.prod(layer.weight.shape[1:]) class GPTQ: @@ -684,8 +683,17 @@ def create_H(self, target_device): return torch.zeros((self.columns, self.columns), dtype=torch.float32, device=self._select_hessian_target_device(target_device)) - def _fallback_quantize(self, strategy: FallbackStrategy, blocksize: int): - """Apply a lightweight quantization fallback using the requested strategy.""" + def _fallback_quantize( + self, + strategy: FallbackStrategy, + blocksize: int, + target_device: Optional[torch.device] = None, + ): + """Apply a lightweight quantization fallback using the requested strategy. + + ``target_device`` is the device the weight clone is quantized on; when + None it is taken from ``self.H`` if present, else the weight. + """ maxq = 2 ** self.qcfg.bits - 1 sigma = 3.0 effective_group_size = self.qcfg.group_size if self.qcfg.group_size != -1 else self.columns @@ -697,7 +705,8 @@ def _fallback_quantize(self, strategy: FallbackStrategy, blocksize: int): mse_steps = smooth_method.steps mse_maxshrink = smooth_method.maxshrink - target_device = self.H.device if self.H is not None else self.module.weight.device + if target_device is None: + target_device = self.H.device if self.H is not None else self.module.weight.device W = self.clone_module(device=target_device) Q = torch.empty_like(W) scale_chunks = [] @@ -989,9 +998,19 @@ def quantize( f"Quantization: Module `{self.name}` -> " f"Using `{resolved_strategy.value}` fallback quantization (observed {self.nsamples} samples, threshold={threshold_text}{threshold_info}, max_total={self.expected_nsamples})." ) - self.H = self.create_H(target_device=target_device) + # The fallback never reads the Hessian: release the fp32 XtX + # partials (columns^2 x 4 B each) instead of folding them into a + # throwaway zero H. Resolve the compute device first, since the + # clear changes what _select_hessian_target_device returns. + with self.lock: + fallback_device = self._select_hessian_target_device(target_device) + self._device_hessian_partials.clear() + self._device_sample_counts.clear() + self._hessian_dirty = False - return self._fallback_quantize(resolved_strategy, blocksize) + return self._fallback_quantize( + resolved_strategy, blocksize, target_device=fallback_device + ) else: use_hessian = True self.finalize_hessian(target_device=target_device) @@ -1446,6 +1465,13 @@ def reset_workspace_stats(self) -> None: self._borrow_workspace_last_chunk_rows = None def free(self): + # The task object outlives free() in processor.tasks until layer + # end, so no path may leave a Hessian partial behind here. + with self.lock: + self._device_hessian_partials.clear() + self._device_sample_counts.clear() + self._hessian_dirty = False + if hasattr(self, "H"): del self.H del self.quantizer diff --git a/gptqmodel/utils/importer.py b/gptqmodel/utils/importer.py index 94c81a28d..e6735f7b8 100644 --- a/gptqmodel/utils/importer.py +++ b/gptqmodel/utils/importer.py @@ -554,7 +554,9 @@ def select_quant_linear( if os.environ.get("DEBUG"): log.info(f"skip {k} for unsupported device `{device}`") continue - supports_sharded_load = getattr(cls, "SUPPORTS_SHARDED_LOAD", cls.SUPPORTS_SHARDS) + supports_sharded_load = getattr( + cls, "SUPPORTS_SHARDED_LOAD", getattr(cls, "SUPPORTS_SHARDS", True) + ) if is_sharded and not supports_sharded_load: if os.environ.get("DEBUG"): log.info(f"skip {k} because sharded checkpoints are not supported") @@ -623,7 +625,9 @@ def select_quant_linear( # Handle the case where backend is not AUTO. qlinear = get_kernel_for_backend(backend, quant_method, format) - supports_sharded_load = getattr(qlinear, "SUPPORTS_SHARDED_LOAD", qlinear.SUPPORTS_SHARDS) + supports_sharded_load = getattr( + qlinear, "SUPPORTS_SHARDED_LOAD", getattr(qlinear, "SUPPORTS_SHARDS", True) + ) if is_sharded and not supports_sharded_load: raise ValueError(f"Selected backend `{backend}` with kernel `{qlinear.__name__}` does not support sharded checkpoints.") diff --git a/tests/kernels/test_selection.py b/tests/kernels/test_selection.py index 406d89899..8672f529d 100644 --- a/tests/kernels/test_selection.py +++ b/tests/kernels/test_selection.py @@ -614,3 +614,36 @@ def test_select_quant_linear_single_select_keeps_model_wide_kernel(monkeypatch): # Marlin/Exllama/Triton do not support base bits=3, so the model-wide kernel # must be TorchLinear, which supports both the base 3-bit and dynamic 4-bit layers. assert selected is TorchLinear + + +def test_sharded_select_tolerates_kernel_without_shard_attrs(monkeypatch): + class BareKernel: + # Defines neither SUPPORTS_SHARDED_LOAD nor SUPPORTS_SHARDS; the + # sharded-load guard must not evaluate a missing attribute eagerly + # and treats absent declarations as shard-capable. + SUPPORTS_DEVICES = [DEVICE.ALL] + + @classmethod + def validate(cls, **_): + return True, None + + monkeypatch.setitem( + AUTO_BACKEND_KERNEL_MAPPING[METHOD.QQQ], + FORMAT.QQQ, + OrderedDict([(BACKEND.QQQ_TORCH, BareKernel)]), + ) + + qlinear_cls = select_quant_linear( + bits=4, + group_size=128, + desc_act=False, + sym=True, + device=torch.device("cpu"), + backend=BACKEND.AUTO, + format=FORMAT.QQQ, + quant_method=METHOD.QQQ, + pack_dtype=torch.int32, + is_sharded=True, + ) + + assert qlinear_cls is BareKernel diff --git a/tests/module_tree/test_ernie_expert_projections.py b/tests/module_tree/test_ernie_expert_projections.py new file mode 100644 index 000000000..f981b15db --- /dev/null +++ b/tests/module_tree/test_ernie_expert_projections.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from gptqmodel.models.definitions.ernie4_5_moe import Ernie4_5_MoeQModel +from gptqmodel.models.definitions.ernie4_5_vl_moe import Ernie4_5_VLMoeQModel + + +def _collect_strings(node): + if isinstance(node, str): + yield node + elif isinstance(node, dict): + for key, value in node.items(): + yield from _collect_strings(key) + yield from _collect_strings(value) + elif isinstance(node, (list, tuple)): + for item in node: + yield from _collect_strings(item) + + +@pytest.mark.parametrize("model_cls", [Ernie4_5_MoeQModel, Ernie4_5_VLMoeQModel]) +def test_expert_projections_use_up_proj(model_cls): + tokens = {entry.split(":", 1)[0] for entry in _collect_strings(model_cls.module_tree)} + # Regression: routed-expert entries carried a "upe_proj" typo while the + # shared-expert entries correctly used "up_proj" (ERNIE 4.5 MoE modeling + # names the expert projections gate_proj/up_proj/down_proj). + assert "upe_proj" not in tokens + assert "up_proj" in tokens diff --git a/tests/test_fallback_hessian_release.py b/tests/test_fallback_hessian_release.py new file mode 100644 index 000000000..e35cdb04e --- /dev/null +++ b/tests/test_fallback_hessian_release.py @@ -0,0 +1,172 @@ +"""The RTN fallback path of GPTQ.quantize() and free() release the Hessian +partials add_batch accumulated; before, only materialize_global_hessian did, +and the task object lingers in processor.tasks until layer end. CPU-only. +""" + +import pytest +import torch +import torch.nn as nn +import transformers + +from gptqmodel.looper.named_module import NamedModule +from gptqmodel.quantization import QuantizeConfig +from gptqmodel.quantization.config import FallbackStrategy +from gptqmodel.quantization.gptq import GPTQ +from gptqmodel.utils.fallback import should_use_fallback + + +COLUMNS = 64 +ROWS = 32 +GROUP_SIZE = 16 +EXPECTED_ROWS = 64 +FALLBACK_ROWS = 8 # < 75% of EXPECTED_ROWS -> fallback branch +SOLVE_ROWS = EXPECTED_ROWS + 64 # >= 75% -> GPTQ solve + + +def _task(*, rows_fed: int, named: bool = True) -> GPTQ: + """A GPTQ task over a tiny Linear fed `rows_fed` calibration rows. + + fallback="75%" of EXPECTED_ROWS, mirroring test_fallback.py + `test_gptq_fallback_threshold_triggers_rtn_when_samples_below_percent`, + plus the NamedModule wrap the looper uses. + """ + torch.manual_seed(0) + linear = nn.Linear(COLUMNS, ROWS, bias=False) + module = linear + if named: + module = NamedModule( + linear, name="mlp.up_proj", full_name="model.layers.0.mlp.up_proj", + layer_index=0, + ) + qcfg = QuantizeConfig(bits=4, group_size=GROUP_SIZE, fallback="75%") + task = GPTQ(module, qcfg) + task.fallback = qcfg.fallback + task.expected_nsamples = EXPECTED_ROWS + task.quantizer.configure(perchannel=True) + task.add_batch(torch.randn(rows_fed, COLUMNS), None) + return task + + +def _assert_partials_released(task: GPTQ) -> None: + assert task._device_hessian_partials == {} + assert task._device_sample_counts == {} + assert task._hessian_dirty is False + + +class TestFallbackReleasesPartials: + def test_fixture_is_on_the_fallback_branch(self): + task = _task(rows_fed=FALLBACK_ROWS) + assert should_use_fallback(task.fallback, float(task.nsamples), task.expected_nsamples) + assert len(task._device_hessian_partials) == 1 + assert sum(task._device_sample_counts.values()) == FALLBACK_ROWS + assert task._hessian_dirty is True + + def test_quantize_fallback_releases_partials(self): + task = _task(rows_fed=FALLBACK_ROWS) + result = task.quantize(blocksize=GROUP_SIZE) + + _assert_partials_released(task) + # nsamples feeds the fallback log line and the returned tuple; the + # release must not touch it. + assert task.nsamples == FALLBACK_ROWS + assert result[7] == FALLBACK_ROWS + assert result[5].startswith("fallback(rtn): ") + + def test_quantize_fallback_matches_pre_fix_arithmetic(self): + """The old branch allocated a zero Hessian with create_H() and let + _fallback_quantize read its device. RTN never reads the Hessian + values, so the result must be identical when the allocation is + skipped and the partials released instead. + """ + baseline = _task(rows_fed=FALLBACK_ROWS) + baseline.H = baseline.create_H(getattr(baseline.module, "target_device", None)) + expected = baseline._fallback_quantize(FallbackStrategy.RTN, GROUP_SIZE) + + task = _task(rows_fed=FALLBACK_ROWS) + actual = task.quantize(blocksize=GROUP_SIZE) + + assert len(actual) == len(expected) + for got, want in zip(actual[:4], expected[:4]): + assert torch.equal(got, want) + assert got.dtype == want.dtype + assert got.device == want.device + # avg_loss, damp, nsamples + assert actual[5] == expected[5] + assert actual[6] == expected[6] + assert actual[7] == expected[7] == FALLBACK_ROWS + + def test_fallback_device_resolved_before_partials_are_cleared(self, monkeypatch): + """The compute device is decided while the partials still exist (the + clear changes what _select_hessian_target_device returns) and is + handed to _fallback_quantize, which sees the dicts already empty. + """ + seen: dict[str, object] = {} + resolve = GPTQ._select_hessian_target_device + + def _spy(self: GPTQ, requested: torch.device | None) -> torch.device: + seen["partials_at_resolve"] = len(self._device_hessian_partials) + return resolve(self, requested) + + def _record( + self: GPTQ, + strategy: FallbackStrategy, + blocksize: int, + target_device: torch.device | None = None, + ) -> tuple: + seen["device"] = target_device + seen["partials_at_entry"] = dict(self._device_hessian_partials) + seen["counts_at_entry"] = dict(self._device_sample_counts) + return ("stub",) * 8 + + monkeypatch.setattr(GPTQ, "_select_hessian_target_device", _spy) + monkeypatch.setattr(GPTQ, "_fallback_quantize", _record) + + task = _task(rows_fed=FALLBACK_ROWS) + task.module.target_device = torch.device("meta") + + assert task.quantize(blocksize=GROUP_SIZE) == ("stub",) * 8 + assert seen["partials_at_resolve"] == 1 + assert seen["device"] == torch.device("meta") + assert seen["partials_at_entry"] == {} + assert seen["counts_at_entry"] == {} + _assert_partials_released(task) + + def test_fallback_quantize_without_device_keeps_old_resolution(self): + task = _task(rows_fed=FALLBACK_ROWS) + task.H = None + result = task._fallback_quantize(FallbackStrategy.RTN, GROUP_SIZE) + assert result[0].device == task.module.weight.device + + def test_gptq_path_still_clears_and_solves(self): + task = _task(rows_fed=SOLVE_ROWS) + result = task.quantize(blocksize=GROUP_SIZE) + _assert_partials_released(task) + assert isinstance(result[5], float) + + +class TestFreeReleasesPartials: + def test_free_clears_partials(self): + task = _task(rows_fed=FALLBACK_ROWS) + assert task._device_hessian_partials + task.free() + _assert_partials_released(task) + + def test_free_on_plain_module_clears_partials(self): + task = _task(rows_fed=FALLBACK_ROWS, named=False) + task.free() + _assert_partials_released(task) + + +class TestColumnsIsPlainInt: + @pytest.mark.parametrize("named", [True, False]) + def test_linear_columns_is_int(self, named): + task = _task(rows_fed=FALLBACK_ROWS, named=named) + assert type(task.columns) is int + assert type(task.rows) is int + assert task.columns == COLUMNS + + def test_conv1d_columns_is_int(self): + conv = transformers.Conv1D(nf=ROWS, nx=COLUMNS) + task = GPTQ(conv, QuantizeConfig(bits=4, group_size=GROUP_SIZE)) + assert type(task.columns) is int + assert task.columns == COLUMNS diff --git a/tests/test_format_gib.py b/tests/test_format_gib.py new file mode 100644 index 000000000..f310013a4 --- /dev/null +++ b/tests/test_format_gib.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from gptqmodel.looper.loop_processor import _format_gib + + +@pytest.mark.parametrize( + "value,expected", + [ + (0.00, "0G"), + (1.00, "1G"), + (1.50, "1.5G"), + (1.25, "1.25G"), + (0.05, "0.05G"), + (10.00, "10G"), + ], +) +def test_format_gib(value, expected): + # Regression: whole-GiB values rendered with a trailing dot ("1.G") + # because only the zeros were stripped from the fixed-point text. + assert _format_gib(value) == expected diff --git a/tests/test_writer_instance_attrs.py b/tests/test_writer_instance_attrs.py new file mode 100644 index 000000000..fccb1287d --- /dev/null +++ b/tests/test_writer_instance_attrs.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-License-Identifier: Apache-2.0 + +import inspect + +from gptqmodel.models.base import BaseQModel + + +def test_get_model_with_quantize_uses_instance_attrs(): + # ModelWriter is applied exactly once, to BaseQModel itself, so inside + # its method bodies the decorator-closure ``cls`` is always BaseQModel. + # Reading ``cls.loader`` / ``cls.lm_head`` there silently discards any + # definition-subclass override (custom loader class, custom lm_head + # name); such attributes must be read from ``self``. + src = inspect.getsource(BaseQModel.get_model_with_quantize) + assert "cls.loader" not in src + assert "cls.lm_head" not in src