From 17078e8d94cc7b030e574441211de355b855b421 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 21 Jul 2026 15:04:53 +0800 Subject: [PATCH 1/5] support inkling Signed-off-by: ZX-ModelCloud --- gptqmodel/models/auto.py | 2 + gptqmodel/models/definitions/__init__.py | 1 + gptqmodel/models/definitions/inkling.py | 153 ++++++++ tests/models/ovis/image_to_test_dataset.py | 2 + tests/models/test_inkling_mm_model.py | 388 +++++++++++++++++++++ 5 files changed, 546 insertions(+) create mode 100644 gptqmodel/models/definitions/inkling.py create mode 100644 tests/models/test_inkling_mm_model.py diff --git a/gptqmodel/models/auto.py b/gptqmodel/models/auto.py index b8cec5f07..00cb240c3 100644 --- a/gptqmodel/models/auto.py +++ b/gptqmodel/models/auto.py @@ -125,6 +125,7 @@ from .definitions.hy_v3 import HYV3QModel # noqa: E402 from .definitions.hymba import HymbaQModel # noqa: E402 from .definitions.instella import InstellaQModel # noqa: E402 +from .definitions.inkling import InklingMMQModel # noqa: E402 from .definitions.internlm import InternLMQModel # noqa: E402 from .definitions.internlm2 import InternLM2QModel # noqa: E402 from .definitions.interns1 import InternS1QModel # noqa: E402 @@ -324,6 +325,7 @@ "ovis2_6_next": Ovis2_6_NextQModel, "telechat": TeleChat2QModel, "instella": InstellaQModel, + "inkling_mm_model": InklingMMQModel, "mimo": MimoQModel, "mimo_v2": MimoV2QModel, "falcon_h1": FalconH1QModel, diff --git a/gptqmodel/models/definitions/__init__.py b/gptqmodel/models/definitions/__init__.py index 9db8c77f3..c918048bb 100644 --- a/gptqmodel/models/definitions/__init__.py +++ b/gptqmodel/models/definitions/__init__.py @@ -51,6 +51,7 @@ from .hy_v3 import HYV3QModel from .hymba import HymbaQModel from .instella import InstellaQModel +from .inkling import InklingMMQModel from .internlm import InternLMQModel from .internlm2 import InternLM2QModel from .interns1 import InternS1QModel diff --git a/gptqmodel/models/definitions/inkling.py b/gptqmodel/models/definitions/inkling.py new file mode 100644 index 000000000..9608531ee --- /dev/null +++ b/gptqmodel/models/definitions/inkling.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from typing import Any, Dict + +import torch +from transformers import AutoModelForMultimodalLM, AutoProcessor, ProcessorMixin +from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask + +from ...utils.calibration import batched +from ...utils.model import MODALITY +from ..base import BaseQModel +from ..moe_lifecycle import GateUpDownMoELifecycleHooks + + +class InklingMMQModel(BaseQModel): + loader = AutoModelForMultimodalLM + + require_load_processor = True + require_trust_remote_code = False + layer_modules_strict = False + + modality = [MODALITY.TEXT, MODALITY.IMAGE_TO_TEXT] + + dynamic_expert_index = "n_routed_experts" + defuser_auto_detect_moe = True + moe_lifecycle_hooks = GateUpDownMoELifecycleHooks() + + pre_lm_head_norm_module = "model.language_model.norm" + + # GQA output shapes differ across the attention projections. AWQ should + # optimize the output projection against its actual attention input. + awq_scale_optimize_shape_dependent_modules = ["self_attn.o_proj"] + + module_tree = [ + "model", + "language_model", + "layers", + "#", + { + "input_layernorm": ("input_layernorm:!",), + "self_attn": ( + "q_proj:0", + "k_proj:0", + "v_proj:0", + "r_proj:0", + "q_norm:!", + "k_norm:!", + "o_proj:1", + ), + "post_attention_layernorm": ("post_attention_layernorm:!",), + "mlp:moe": { + # Inkling's first decoder blocks use a dense MLP. + "": ("gate_proj:0", "up_proj:0", "down_proj:1"), + "gate": ("gate:!",), + "experts": { + "#": ("gate_proj:0", "up_proj:0", "down_proj:1"), + }, + # Shared experts are native 3D parameters rather than Linear + # modules. Keep these two accuracy-sensitive experts dense; + # routed experts are expanded and quantized individually. + }, + }, + ] + + def load_processor(self) -> ProcessorMixin: + return AutoProcessor.from_pretrained(self.model_local_path, trust_remote_code=False) + + @classmethod + def prepare_inputs_for_conversations( + cls, + processor: ProcessorMixin, + conversations: list[dict] | list[list[dict]], + ): + return processor.apply_chat_template( + conversations, + tokenize=True, + add_generation_prompt=True, + reasoning_effort="medium", + return_dict=True, + return_tensors="pt", + ) + + def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): + del kwargs + processor = self.load_processor() + calibration_data = [] + for batch in batched(calibration_dataset, batch_size): + calibration_data.append(self.prepare_inputs_for_conversations(processor, batch)) + del processor + return calibration_data + + def move_input_capture_example( + self, + example: Dict[str, Any], + data_device: torch.device, + ) -> Dict[str, Any]: + example = super().move_input_capture_example(example, data_device) + pixel_values = example.get("pixel_values") + if not torch.is_tensor(pixel_values) or not pixel_values.is_floating_point(): + return example + + first_parameter = next(self.model.model.vision_tower.parameters(), None) + if first_parameter is not None: + example["pixel_values"] = pixel_values.to( + device=first_parameter.device, + dtype=first_parameter.dtype, + ) + return example + + def prepare_layer_replay_kwargs(self, layer, layer_input, additional_inputs, target_device): + additional_inputs = super().prepare_layer_replay_kwargs( + layer, + layer_input, + additional_inputs, + target_device, + ) + if not layer_input or not torch.is_tensor(layer_input[0]): + return additional_inputs + + self_attn = getattr(layer, "self_attn", None) + layer_config = getattr(self_attn, "config", None) + if layer_config is None: + return additional_inputs + + # The first-layer hook captures the sliding layer's prepared 4D mask. + # Rebuild it for every replayed layer so full-attention blocks do not + # accidentally inherit the sliding window. ``conv_mask`` retains the + # original 2D padding signal when the calibration batch is padded. + padding_mask = additional_inputs.get("conv_mask") + if not torch.is_tensor(padding_mask) or padding_mask.ndim != 2: + captured_mask = additional_inputs.get("attention_mask") + padding_mask = captured_mask if torch.is_tensor(captured_mask) and captured_mask.ndim == 2 else None + + mask_factory = ( + create_sliding_window_causal_mask + if getattr(layer, "layer_type", None) == "hybrid_sliding" + else create_causal_mask + ) + additional_inputs["attention_mask"] = mask_factory( + config=layer_config, + inputs_embeds=layer_input[0], + attention_mask=padding_mask, + past_key_values=additional_inputs.get("past_key_values"), + position_ids=additional_inputs.get("position_ids"), + layer_idx=getattr(self_attn, "layer_idx", None), + ) + return additional_inputs + + +__all__ = ["InklingMMQModel"] diff --git a/tests/models/ovis/image_to_test_dataset.py b/tests/models/ovis/image_to_test_dataset.py index f7ed51ec4..6f7e3df6c 100644 --- a/tests/models/ovis/image_to_test_dataset.py +++ b/tests/models/ovis/image_to_test_dataset.py @@ -8,6 +8,7 @@ from gptqmodel.models.definitions.deepseek_vl import DeepSeekVLQModel from gptqmodel.models.definitions.deepseek_vl_v2 import DeepSeekVLV2QModel from gptqmodel.models.definitions.ernie4_5_vl_moe import Ernie4_5_VLMoeQModel +from gptqmodel.models.definitions.inkling import InklingMMQModel from gptqmodel.models.definitions.interns1 import InternS1QModel from gptqmodel.models.definitions.internvl_chat import InternVLChatQModel from gptqmodel.models.definitions.lfm2_vl import LFM2VLQModel @@ -164,6 +165,7 @@ def get_calib_dataset(model): or isinstance(model, MiniCPMV4_6QModel) or isinstance(model, InternS1QModel) or isinstance(model, InternVLChatQModel) + or isinstance(model, InklingMMQModel) or isinstance(model, Ernie4_5_VLMoeQModel) or isinstance(model, LFM2VLQModel) ): diff --git a/tests/models/test_inkling_mm_model.py b/tests/models/test_inkling_mm_model.py new file mode 100644 index 000000000..3427f8780 --- /dev/null +++ b/tests/models/test_inkling_mm_model.py @@ -0,0 +1,388 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +import json +import unittest +from pathlib import Path +from types import SimpleNamespace + +import torch +from model_test import ModelTest +from safetensors.torch import save_file +from torch import nn +from transformers import AutoModelForMultimodalLM, InklingConfig +from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask + +from gptqmodel.models import auto +from gptqmodel.models.definitions.inkling import InklingMMQModel +from gptqmodel.models.loader import _convert_model_with_defuser +from gptqmodel.utils.structure import LazyTurtle + + +MODEL_PATH = Path("/monster/data/model/Inkling") + + +def _tiny_config() -> InklingConfig: + return InklingConfig( + text_config={ + "hidden_size": 16, + "num_hidden_layers": 3, + "layer_types": ["hybrid_sliding", "hybrid_sliding", "hybrid"], + "mlp_layer_types": ["dense", "sparse", "sparse"], + "num_attention_heads": 2, + "num_key_value_heads": 1, + "head_dim": 8, + "swa_num_attention_heads": 2, + "swa_num_key_value_heads": 1, + "swa_head_dim": 8, + "vocab_size": 64, + "intermediate_size": 12, + "moe_intermediate_size": 6, + "n_routed_experts": 2, + "num_experts_per_tok": 1, + "n_shared_experts": 1, + "d_rel": 2, + "rel_extent": 8, + "sliding_window_size": 4, + "conv_kernel_size": 2, + "max_position_embeddings": 32, + "pad_token_id": 0, + "bos_token_id": 1, + "eos_token_id": 2, + }, + audio_config={"n_mel_bins": 4, "mel_vocab_size": 8}, + vision_config={ + "hidden_size": 8, + "num_hidden_layers": 1, + "num_attention_heads": 2, + "patch_size": 4, + "temporal_patch_size": 2, + }, + image_token_id=60, + audio_token_id=61, + image_bos_token_id=62, + audio_bos_token_id=63, + ) + + +def _tiny_model(): + torch.manual_seed(0) + model = AutoModelForMultimodalLM.from_config(_tiny_config(), dtype=torch.float32) + # The optimized grouped-mm backend is CUDA-only; use the model's native + # eager expert loop for the CPU equivalence assertion. + model.config.text_config._experts_implementation = "eager" + model.eval() + return model + + +def _interleave(tensor: torch.Tensor, dim: int) -> torch.Tensor: + shape = list(tensor.shape) + shape[dim : dim + 1] = [shape[dim] // 2, 2] + return tensor.reshape(shape).transpose(dim, dim + 1).reshape(tensor.shape).contiguous() + + +def _complete_local_checkpoint() -> bool: + index_path = MODEL_PATH / "model.safetensors.index.json" + if not index_path.is_file(): + return False + try: + weight_map = json.loads(index_path.read_text(encoding="utf-8"))["weight_map"] + except (KeyError, OSError, json.JSONDecodeError): + return False + return bool(weight_map) and all((MODEL_PATH / filename).is_file() for filename in set(weight_map.values())) + + +def test_inkling_model_type_selects_definition(monkeypatch): + fake_config = SimpleNamespace(model_type="inkling_mm_model") + + monkeypatch.setattr(auto, "resolve_trust_remote_code", lambda path, trust_remote_code=False: trust_remote_code) + monkeypatch.setattr(auto.AutoConfig, "from_pretrained", lambda *args, **kwargs: fake_config) + + assert auto.check_and_get_model_definition("/tmp/inkling") is InklingMMQModel + + +def test_inkling_module_tree_covers_attention_dense_and_routed_experts(): + layer_modules = InklingMMQModel.simple_layer_modules( + model_config=SimpleNamespace(text_config=SimpleNamespace(n_routed_experts=2)), + quantize_config=SimpleNamespace(dynamic=None), + ) + flat_modules = {name for block in layer_modules for name in block} + + assert InklingMMQModel.loader is AutoModelForMultimodalLM + assert InklingMMQModel.layer_modules_strict is False + assert InklingMMQModel.defuser_auto_detect_moe is True + assert InklingMMQModel.extract_layers_node() == ["model.language_model.layers"] + assert InklingMMQModel.pre_lm_head_norm_module == "model.language_model.norm" + assert "self_attn.q_proj" in flat_modules + assert "self_attn.k_proj" in flat_modules + assert "self_attn.v_proj" in flat_modules + assert "self_attn.r_proj" in flat_modules + assert "self_attn.o_proj" in flat_modules + assert "mlp.gate_proj" in flat_modules + assert "mlp.up_proj" in flat_modules + assert "mlp.down_proj" in flat_modules + assert "mlp.experts.0.gate_proj" in flat_modules + assert "mlp.experts.1.up_proj" in flat_modules + assert "mlp.experts.0.down_proj" in flat_modules + assert not any("shared_experts" in name for name in flat_modules) + + +def test_inkling_calibration_uses_native_multimodal_chat_template(): + class RecordingProcessor: + def apply_chat_template(self, conversations, **kwargs): + self.conversations = conversations + self.kwargs = kwargs + return {"input_ids": torch.tensor([[1, 2]])} + + processor = RecordingProcessor() + conversations = [{"role": "user", "content": [{"type": "text", "text": "Describe the image."}]}] + + result = InklingMMQModel.prepare_inputs_for_conversations(processor, conversations) + + assert result["input_ids"].shape == (1, 2) + assert processor.conversations is conversations + assert processor.kwargs == { + "tokenize": True, + "add_generation_prompt": True, + "reasoning_effort": "medium", + "return_dict": True, + "return_tensors": "pt", + } + + +def test_inkling_defuser_expands_packed_routed_experts_without_changing_forward(): + model = _tiny_model() + input_ids = torch.tensor([[1, 8, 9, 2]]) + packed_experts = model.model.language_model.layers[1].mlp.experts + shared_experts = model.model.language_model.layers[1].mlp.shared_experts + + assert hasattr(packed_experts, "gate_up_proj") + assert isinstance(shared_experts.gate_proj, nn.Parameter) + with torch.inference_mode(): + expected = model(input_ids=input_ids, use_cache=False).logits + + assert _convert_model_with_defuser(InklingMMQModel, model, cleanup_original=False) is True + + experts = model.model.language_model.layers[1].mlp.experts + assert not hasattr(experts, "gate_up_proj") + assert isinstance(experts[0].gate_proj, nn.Linear) + assert isinstance(experts[0].up_proj, nn.Linear) + assert isinstance(experts[0].down_proj, nn.Linear) + assert isinstance(shared_experts.gate_proj, nn.Parameter) + with torch.inference_mode(): + actual = model(input_ids=input_ids, use_cache=False).logits + + torch.testing.assert_close(actual, expected, rtol=1e-6, atol=1e-8) + + +def test_inkling_replay_rebuilds_sliding_and_full_attention_masks(): + model = _tiny_model() + wrapper = InklingMMQModel.__new__(InklingMMQModel) + hidden_states = torch.zeros(1, 6, model.config.text_config.hidden_size) + padding_mask = torch.tensor([[1, 1, 1, 1, 0, 0]], dtype=torch.bool) + sliding_layer = model.model.language_model.layers[0] + full_layer = model.model.language_model.layers[2] + + mask_kwargs = { + "config": model.config.text_config, + "inputs_embeds": hidden_states, + "attention_mask": padding_mask, + "past_key_values": None, + "position_ids": None, + } + captured_sliding_mask = create_sliding_window_causal_mask( + **mask_kwargs, + layer_idx=sliding_layer.self_attn.layer_idx, + ) + base_inputs = { + "attention_mask": captured_sliding_mask, + "conv_mask": padding_mask, + "past_key_values": None, + } + + sliding_inputs = wrapper.prepare_layer_replay_kwargs( + sliding_layer, + [hidden_states], + dict(base_inputs), + torch.device("cpu"), + ) + full_inputs = wrapper.prepare_layer_replay_kwargs( + full_layer, + [hidden_states], + dict(base_inputs), + torch.device("cpu"), + ) + expected_full_mask = create_causal_mask( + **mask_kwargs, + layer_idx=full_layer.self_attn.layer_idx, + ) + + assert torch.equal(sliding_inputs["attention_mask"], captured_sliding_mask) + assert torch.equal(full_inputs["attention_mask"], expected_full_mask) + assert not torch.equal(full_inputs["attention_mask"], captured_sliding_mask) + assert full_inputs["conv_mask"] is padding_mask + + +def test_inkling_lazy_turtle_applies_interleave_before_dense_and_expert_splits(tmp_path): + model = _tiny_model() + assert _convert_model_with_defuser(InklingMMQModel, model, cleanup_original=False) is True + + dense_w13 = torch.arange(24 * 16, dtype=torch.float32).reshape(24, 16) + routed_w13 = torch.arange(2 * 12 * 16, dtype=torch.float32).reshape(2, 12, 16) + shared_w13 = torch.arange(12 * 16, dtype=torch.float32).reshape(1, 12, 16) + save_file( + { + "model.llm.layers.0.mlp.w13_dn.weight": dense_w13, + "model.llm.layers.1.mlp.experts.w13_weight": routed_w13, + "model.llm.layers.1.mlp.shared_experts.shared_w13_weight": shared_w13, + }, + tmp_path / "model.safetensors", + ) + turtle = LazyTurtle( + model_local_path=str(tmp_path), + config=model.config, + module_tree=InklingMMQModel.module_tree, + hf_conversion_map_reversed=InklingMMQModel.resolve_hf_conversion_map_reversed(target_model=model), + target_model=model, + ) + + dense_source = turtle._resolve_checkpoint_tensor_source( + "model.language_model.layers.0.mlp.gate_proj", + "weight", + ) + routed_source = turtle._resolve_checkpoint_tensor_source( + "model.language_model.layers.1.mlp.experts.1.up_proj", + "weight", + ) + shared_source = turtle._resolve_checkpoint_tensor_source( + "model.language_model.layers.1.mlp.shared_experts", + "gate_proj", + ) + + assert dense_source[:3] == ("model.llm.layers.0.mlp.w13_dn.weight", None, 0) + assert routed_source[:3] == ("model.llm.layers.1.mlp.experts.w13_weight", 1, 1) + assert shared_source[:3] == ( + "model.llm.layers.1.mlp.shared_experts.shared_w13_weight", + None, + 0, + ) + assert getattr(dense_source[3], "interleave_dim", None) == 0 + assert getattr(routed_source[3], "interleave_dim", None) == 1 + assert getattr(shared_source[3], "interleave_dim", None) == 1 + + dense_gate = turtle._transform_checkpoint_tensor( + dense_w13, + expert_index=dense_source[1], + split_index=dense_source[2], + split_dim=dense_source[3], + expected_shape=(12, 16), + ) + routed_up = turtle._transform_checkpoint_tensor( + routed_w13, + expert_index=routed_source[1], + split_index=routed_source[2], + split_dim=routed_source[3], + expected_shape=(6, 16), + ) + shared_gate = turtle._transform_checkpoint_tensor( + shared_w13, + expert_index=shared_source[1], + split_index=shared_source[2], + split_dim=shared_source[3], + expected_shape=(1, 6, 16), + ) + + expected_dense_gate = _interleave(dense_w13, dim=0).chunk(2, dim=0)[0] + expected_routed_up = _interleave(routed_w13[1], dim=0).chunk(2, dim=0)[1] + expected_shared_gate = _interleave(shared_w13, dim=1).chunk(2, dim=1)[0] + assert torch.equal(dense_gate, expected_dense_gate) + assert torch.equal(routed_up, expected_routed_up) + assert torch.equal(shared_gate, expected_shared_gate) + + # Exercise the real shell-materialization path as well as the resolver: + # these are the three target layouts Inkling exposes after model creation + # and routed-expert defusion. + modules = dict(model.named_modules()) + materialization_cases = ( + ( + modules["model.language_model.layers.0.mlp.gate_proj"], + expected_dense_gate, + ), + ( + modules["model.language_model.layers.1.mlp.experts.1.up_proj"], + expected_routed_up, + ), + ) + for target_module, expected_weight in materialization_cases: + target_module.weight = nn.Parameter(torch.empty_like(target_module.weight, device="meta")) + turtle.materialize_direct_meta_tensors( + target_model=model, + target_submodule=target_module, + device=torch.device("cpu"), + ) + assert torch.equal(target_module.weight, expected_weight) + + shared_experts = modules["model.language_model.layers.1.mlp.shared_experts"] + shared_experts.gate_proj = nn.Parameter(torch.empty_like(shared_experts.gate_proj, device="meta")) + turtle.materialize_direct_meta_tensors( + target_model=model, + target_submodule=shared_experts, + device=torch.device("cpu"), + ) + assert torch.equal(shared_experts.gate_proj, expected_shared_gate) + + +@unittest.skipUnless(_complete_local_checkpoint(), "Complete local Inkling checkpoint not found") +class TestInklingMMModel(ModelTest): + NATIVE_MODEL_ID = str(MODEL_PATH) + TRUST_REMOTE_CODE = False + USE_FLASH_ATTN = False + EVAL_BATCH_SIZE = 1 + MODEL_COMPAT_FAST_LAYER_POSITION = "first" + MODEL_COMPAT_FAST_LAYER_COUNT = 3 + + def test_inkling_mm_model(self): + with self.model_compat_test_context(): + model, _tokenizer, processor = self.quantModel( + self.NATIVE_MODEL_ID, + trust_remote_code=self.TRUST_REMOTE_CODE, + dtype=self.TORCH_DTYPE, + batch_size=1, + call_perform_post_quant_validation=False, + ) + + image_url = ( + "https://huggingface.co/datasets/merve/vl-test-suite/" + "resolve/main/pills.jpg" + ) + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": image_url}, + {"type": "text", "text": "Do any of the components in this supplement interact?"}, + ], + }, + ] + inputs = processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + reasoning_effort="medium", + return_dict=True, + return_tensors="pt", + ).to(model.device) + input_length = inputs["input_ids"].shape[-1] + + outputs = model.generate(**inputs, max_new_tokens=512) + response = processor.decode(outputs[0][input_length:], skip_special_tokens=False) + parsed_response = processor.parse_response(response) + + self.assertTrue(response) + self.assertIsInstance(parsed_response, dict) + self.assertTrue(parsed_response) + + +__all__ = ["TestInklingMMModel"] From 8b20609ad7d38fe4bf63cea20488d71dd1c4cc12 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 21 Jul 2026 20:27:40 +0800 Subject: [PATCH 2/5] fix test_inkling_mm_model.py Signed-off-by: ZX-ModelCloud --- gptqmodel/models/definitions/inkling.py | 43 +++++++++++- tests/models/test_inkling_mm_model.py | 91 ++++++++++--------------- 2 files changed, 78 insertions(+), 56 deletions(-) diff --git a/gptqmodel/models/definitions/inkling.py b/gptqmodel/models/definitions/inkling.py index 9608531ee..ac71c3455 100644 --- a/gptqmodel/models/definitions/inkling.py +++ b/gptqmodel/models/definitions/inkling.py @@ -10,7 +10,9 @@ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...utils.calibration import batched -from ...utils.model import MODALITY +from ...utils.model import MODALITY, move_to +from ...utils.offload import offload_to_disk +from .._const import CPU from ..base import BaseQModel from ..moe_lifecycle import GateUpDownMoELifecycleHooks @@ -65,6 +67,45 @@ class InklingMMQModel(BaseQModel): }, ] + def pre_quantize_generate_hook_start(self): + core_model = self.model.model + language_model = core_model.language_model + self.shell_module_materialize(language_model.embed_tokens, self.quantize_config.device) + self.shell_module_materialize(language_model.embed_norm, self.quantize_config.device) + self.shell_module_materialize(core_model.vision_tower, self.quantize_config.device) + self.shell_module_materialize(core_model.audio_tower, self.quantize_config.device) + + def pre_quantize_generate_hook_end(self): + core_model = self.model.model + language_model = core_model.language_model + if self.quantize_config.offload_to_disk: + offload_to_disk( + model=language_model, + module=language_model.embed_tokens, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=language_model, + module=language_model.embed_norm, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=core_model, + module=core_model.vision_tower, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=core_model, + module=core_model.audio_tower, + disk_path=self.quantize_config.offload_to_disk_path, + ) + return + + language_model.embed_tokens = move_to(language_model.embed_tokens, device=CPU) + language_model.embed_norm = move_to(language_model.embed_norm, device=CPU) + core_model.vision_tower = move_to(core_model.vision_tower, device=CPU) + core_model.audio_tower = move_to(core_model.audio_tower, device=CPU) + def load_processor(self) -> ProcessorMixin: return AutoProcessor.from_pretrained(self.model_local_path, trust_remote_code=False) diff --git a/tests/models/test_inkling_mm_model.py b/tests/models/test_inkling_mm_model.py index 3427f8780..e6f619e66 100644 --- a/tests/models/test_inkling_mm_model.py +++ b/tests/models/test_inkling_mm_model.py @@ -3,8 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 # Contact: qubitium@modelcloud.ai, x.com/qubitium -import json -import unittest from pathlib import Path from types import SimpleNamespace @@ -21,7 +19,7 @@ from gptqmodel.utils.structure import LazyTurtle -MODEL_PATH = Path("/monster/data/model/Inkling") +MODEL_PATH = Path("/monster/data/model/Inkling-0.6B-A0.6B-BF16") def _tiny_config() -> InklingConfig: @@ -83,17 +81,6 @@ def _interleave(tensor: torch.Tensor, dim: int) -> torch.Tensor: return tensor.reshape(shape).transpose(dim, dim + 1).reshape(tensor.shape).contiguous() -def _complete_local_checkpoint() -> bool: - index_path = MODEL_PATH / "model.safetensors.index.json" - if not index_path.is_file(): - return False - try: - weight_map = json.loads(index_path.read_text(encoding="utf-8"))["weight_map"] - except (KeyError, OSError, json.JSONDecodeError): - return False - return bool(weight_map) and all((MODEL_PATH / filename).is_file() for filename in set(weight_map.values())) - - def test_inkling_model_type_selects_definition(monkeypatch): fake_config = SimpleNamespace(model_type="inkling_mm_model") @@ -152,6 +139,32 @@ def apply_chat_template(self, conversations, **kwargs): } +def test_inkling_multimodal_capture_materializes_preforward_modules_on_quant_device(): + class RecordingInklingMMQModel(InklingMMQModel): + def shell_module_materialize(self, target_submodule, device, **kwargs): + del kwargs + self.materialize_calls.append((target_submodule, device)) + return target_submodule + + model = _tiny_model() + wrapper = RecordingInklingMMQModel.__new__(RecordingInklingMMQModel) + nn.Module.__init__(wrapper) + wrapper.model = model + wrapper.quantize_config = SimpleNamespace(device=torch.device("cuda:0")) + wrapper.materialize_calls = [] + + wrapper.pre_quantize_generate_hook_start() + + core_model = model.model + assert [module for module, _device in wrapper.materialize_calls] == [ + core_model.language_model.embed_tokens, + core_model.language_model.embed_norm, + core_model.vision_tower, + core_model.audio_tower, + ] + assert all(device == torch.device("cuda:0") for _module, device in wrapper.materialize_calls) + + def test_inkling_defuser_expands_packed_routed_experts_without_changing_forward(): model = _tiny_model() input_ids = torch.tensor([[1, 8, 9, 2]]) @@ -334,55 +347,23 @@ def test_inkling_lazy_turtle_applies_interleave_before_dense_and_expert_splits(t assert torch.equal(shared_experts.gate_proj, expected_shared_gate) -@unittest.skipUnless(_complete_local_checkpoint(), "Complete local Inkling checkpoint not found") class TestInklingMMModel(ModelTest): NATIVE_MODEL_ID = str(MODEL_PATH) TRUST_REMOTE_CODE = False USE_FLASH_ATTN = False EVAL_BATCH_SIZE = 1 MODEL_COMPAT_FAST_LAYER_POSITION = "first" - MODEL_COMPAT_FAST_LAYER_COUNT = 3 + EVAL_TASKS_SLOW = { + "arc_challenge": { + "chat_template": True, + "acc": {"value": 0.2098, "floor_pct": 0.04}, + "acc_norm": {"value": 0.2389, "floor_pct": 0.04}, + }, + } + EVAL_TASKS_FAST = ModelTest.derive_fast_eval_tasks(EVAL_TASKS_SLOW) def test_inkling_mm_model(self): - with self.model_compat_test_context(): - model, _tokenizer, processor = self.quantModel( - self.NATIVE_MODEL_ID, - trust_remote_code=self.TRUST_REMOTE_CODE, - dtype=self.TORCH_DTYPE, - batch_size=1, - call_perform_post_quant_validation=False, - ) - - image_url = ( - "https://huggingface.co/datasets/merve/vl-test-suite/" - "resolve/main/pills.jpg" - ) - messages = [ - { - "role": "user", - "content": [ - {"type": "image", "image": image_url}, - {"type": "text", "text": "Do any of the components in this supplement interact?"}, - ], - }, - ] - inputs = processor.apply_chat_template( - messages, - tokenize=True, - add_generation_prompt=True, - reasoning_effort="medium", - return_dict=True, - return_tensors="pt", - ).to(model.device) - input_length = inputs["input_ids"].shape[-1] - - outputs = model.generate(**inputs, max_new_tokens=512) - response = processor.decode(outputs[0][input_length:], skip_special_tokens=False) - parsed_response = processor.parse_response(response) - - self.assertTrue(response) - self.assertIsInstance(parsed_response, dict) - self.assertTrue(parsed_response) + self.quantize_and_evaluate() __all__ = ["TestInklingMMModel"] From 6bbe4c08e78180bb66374b4c7ad3fec0fa54730e Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 21 Jul 2026 23:58:26 +0800 Subject: [PATCH 3/5] fix LazyTurtle Signed-off-by: ZX-ModelCloud --- gptqmodel/utils/structure.py | 114 ++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 9 deletions(-) diff --git a/gptqmodel/utils/structure.py b/gptqmodel/utils/structure.py index 00015fd20..667a0c221 100644 --- a/gptqmodel/utils/structure.py +++ b/gptqmodel/utils/structure.py @@ -558,6 +558,31 @@ def _ensure_target_storage_on_device_(param: torch.nn.Parameter, device: torch.d return param +class _SplitDimWithInterleave(int): + """Integer split dimension carrying a preceding HF Interleave operation. + + Keeping this as an ``int`` preserves the private resolver's four-item return + contract while allowing materialization to reproduce checkpoint conversion + pipelines such as Inkling's ``Interleave -> Chunk``. + """ + + def __new__(cls, split_dim: int, *, interleave_dim: int, interleave_inverse: bool): + value = super().__new__(cls, split_dim) + value.interleave_dim = interleave_dim + value.interleave_inverse = interleave_inverse + return value + + +def _split_dim_cache_key(split_dim: Optional[int]) -> tuple[Optional[int], Optional[int], Optional[bool]]: + if split_dim is None: + return None, None, None + return ( + int(split_dim), + getattr(split_dim, "interleave_dim", None), + getattr(split_dim, "interleave_inverse", None), + ) + + @dataclass(frozen=True) class _MoEAliasSpec: """MoE alias groups derived entirely from the model definition's `module_tree`.""" @@ -1797,6 +1822,10 @@ def _resolve_converter_tensor_source( fused_candidate = (fused_name, expert_index, split_index, split_dim) if fused_candidate not in runtime_candidates: runtime_candidates.append(fused_candidate) + for fused_name, expert_index, split_index, split_dim in self._fused_checkpoint_requests(combined_name): + fused_candidate = (fused_name, expert_index, split_index, split_dim) + if fused_candidate not in runtime_candidates: + runtime_candidates.append(fused_candidate) for converter in self._runtime_to_checkpoint_converters: if "ErnieFuseAndSplitTextVisionExperts" in converter.operation_names: @@ -1833,15 +1862,47 @@ def _resolve_converter_tensor_source( split_index = None split_dim = None - if converter.operation_names[:1] == ("Chunk",) and len(converter.source_patterns) > 1: + resolved_expert_index = None + if len(converter.target_patterns) == 1: + resolved_expert_index = fused_expert_index + split_index = fused_split_index + split_dim = fused_split_dim + + chunk_operation = next( + ( + operation + for operation in converter.operations + if type(operation).__name__ == "Chunk" + ), + None, + ) + if chunk_operation is not None and len(converter.source_patterns) > 1: # Chunk converters share one checkpoint tensor across # multiple runtime tensors; preserve the selected slice. split_index = selected_source_index - split_dim = getattr(converter.operations[0], "dim", 0) + split_dim = getattr(chunk_operation, "dim", 0) + + interleave_operation = next( + ( + operation + for operation in converter.operations + if type(operation).__name__ == "Interleave" + ), + None, + ) + if interleave_operation is not None and split_dim is not None: + split_dim = _SplitDimWithInterleave( + split_dim, + interleave_dim=getattr(interleave_operation, "dim", 0), + interleave_inverse=bool(getattr(interleave_operation, "inverse", False)), + ) renamed_variants = [renamed] if "*" in renamed and fused_expert_index is not None: renamed_variants.insert(0, renamed.replace("*", str(fused_expert_index), 1)) + # The expert index is encoded in the checkpoint key, not + # stored as the leading axis of the checkpoint tensor. + resolved_expert_index = None for renamed_variant in renamed_variants: for candidate in self._runtime_to_checkpoint_alias_candidates(renamed_variant): @@ -1851,7 +1912,7 @@ def _resolve_converter_tensor_source( )[0] is not None: continue if candidate in self._weight_map: - return candidate, None, split_index, split_dim + return candidate, resolved_expert_index, split_index, split_dim return None, None, None, None @@ -1956,18 +2017,53 @@ def _transform_checkpoint_tensor( ) -> Optional[torch.Tensor]: """Slice fused checkpoint tensors into the tensor layout expected by the shell module.""" + interleave_dim = getattr(split_dim, "interleave_dim", None) + interleave_inverse = bool(getattr(split_dim, "interleave_inverse", False)) + effective_split_dim = int(split_dim) if split_dim is not None else None + + def apply_interleave(candidate: torch.Tensor, dim: int) -> Optional[torch.Tensor]: + if dim < 0: + dim += candidate.ndim + if dim < 0 or dim >= candidate.ndim or candidate.shape[dim] % 2: + return None + + shape = list(candidate.shape) + if interleave_inverse: + shape[dim : dim + 1] = [2, shape[dim] // 2] + else: + shape[dim : dim + 1] = [shape[dim] // 2, 2] + return candidate.reshape(shape).transpose(dim, dim + 1).reshape(candidate.shape).contiguous() + + # If a converter acts on the leading expert axis, it must run before + # selecting an expert. Other dimensions can be adjusted after slicing, + # avoiding a copy of the complete packed expert tensor. + if expert_index is not None and interleave_dim == 0: + tensor = apply_interleave(tensor, interleave_dim) + if tensor is None: + return None + interleave_dim = None + if expert_index is not None: if tensor.shape[0] <= expert_index: return None # Fused expert checkpoints store the expert axis first; peel it off before # reasoning about split dimensions or transpose decisions. tensor = tensor[expert_index].contiguous() + if interleave_dim is not None and interleave_dim > 0: + interleave_dim -= 1 + if effective_split_dim is not None and effective_split_dim > 0: + effective_split_dim -= 1 + + if interleave_dim is not None: + tensor = apply_interleave(tensor, interleave_dim) + if tensor is None: + return None if expected_shape is None: if split_index is not None: - if split_dim is None or tensor.shape[split_dim] % 2 != 0: + if effective_split_dim is None or tensor.shape[effective_split_dim] % 2 != 0: return None - tensor = tensor.chunk(2, dim=split_dim)[split_index].contiguous() + tensor = tensor.chunk(2, dim=effective_split_dim)[split_index].contiguous() return tensor expected_shape = tuple(expected_shape) @@ -1979,16 +2075,16 @@ def try_candidate(candidate: torch.Tensor, *, used_transpose: bool) -> Optional[ return None preferred_dims: list[int] = [] - mapped_split_dim = split_dim + mapped_split_dim = effective_split_dim if ( used_transpose and candidate.ndim == 2 - and split_dim is not None - and 0 <= split_dim < 2 + and effective_split_dim is not None + and 0 <= effective_split_dim < 2 ): # The resolver hint is expressed in the checkpoint's native layout. # Once we transpose a 2D candidate, the split dimension flips too. - mapped_split_dim = 1 - split_dim + mapped_split_dim = 1 - effective_split_dim if mapped_split_dim is not None and 0 <= mapped_split_dim < candidate.ndim: preferred_dims.append(mapped_split_dim) preferred_dims.extend(dim for dim in range(candidate.ndim) if dim not in preferred_dims) From 71fd4aad3a66f05b6915c9f979047d474e929c48 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Wed, 22 Jul 2026 10:35:27 +0800 Subject: [PATCH 4/5] cleanup Signed-off-by: ZX-ModelCloud --- gptqmodel/utils/structure.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gptqmodel/utils/structure.py b/gptqmodel/utils/structure.py index 667a0c221..e1573d4c6 100644 --- a/gptqmodel/utils/structure.py +++ b/gptqmodel/utils/structure.py @@ -574,6 +574,7 @@ def __new__(cls, split_dim: int, *, interleave_dim: int, interleave_inverse: boo def _split_dim_cache_key(split_dim: Optional[int]) -> tuple[Optional[int], Optional[int], Optional[bool]]: + """Keep Interleave metadata distinct when a split dimension is used as a cache key.""" if split_dim is None: return None, None, None return ( @@ -1822,6 +1823,7 @@ def _resolve_converter_tensor_source( fused_candidate = (fused_name, expert_index, split_index, split_dim) if fused_candidate not in runtime_candidates: runtime_candidates.append(fused_candidate) + # The expert index may be part of module_path, so inspect the full tensor name too. for fused_name, expert_index, split_index, split_dim in self._fused_checkpoint_requests(combined_name): fused_candidate = (fused_name, expert_index, split_index, split_dim) if fused_candidate not in runtime_candidates: @@ -1864,6 +1866,7 @@ def _resolve_converter_tensor_source( split_dim = None resolved_expert_index = None if len(converter.target_patterns) == 1: + # One checkpoint tensor can hold every expert and both gate/up projections. resolved_expert_index = fused_expert_index split_index = fused_split_index split_dim = fused_split_dim @@ -1891,6 +1894,7 @@ def _resolve_converter_tensor_source( None, ) if interleave_operation is not None and split_dim is not None: + # Inkling deinterleaves w13 before selecting the gate/up slice. split_dim = _SplitDimWithInterleave( split_dim, interleave_dim=getattr(interleave_operation, "dim", 0), @@ -2022,6 +2026,7 @@ def _transform_checkpoint_tensor( effective_split_dim = int(split_dim) if split_dim is not None else None def apply_interleave(candidate: torch.Tensor, dim: int) -> Optional[torch.Tensor]: + """Apply HF's two-way Interleave operation without changing tensor shape.""" if dim < 0: dim += candidate.ndim if dim < 0 or dim >= candidate.ndim or candidate.shape[dim] % 2: @@ -2049,6 +2054,7 @@ def apply_interleave(candidate: torch.Tensor, dim: int) -> Optional[torch.Tensor # Fused expert checkpoints store the expert axis first; peel it off before # reasoning about split dimensions or transpose decisions. tensor = tensor[expert_index].contiguous() + # Removing the expert axis shifts positive converter dimensions left. if interleave_dim is not None and interleave_dim > 0: interleave_dim -= 1 if effective_split_dim is not None and effective_split_dim > 0: From 021420acd06ab4ac50b1fdfacbbb4a0eb28c6d57 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Thu, 23 Jul 2026 11:31:39 +0800 Subject: [PATCH 5/5] update README.md Signed-off-by: ZX-ModelCloud --- README.md | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2477d77bc..c6eebe101 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ ## Latest News +* 07/23/2026 7.3.1 `main`: ✨ Added `inkling_mm_model` model support * 07/22/2026 7.3.1 `main`: ✨ Added Poolside `Laguna S 2.1` model support * 07/14/2026 7.3.0-dev `main`: ✨ Added `nemotron_h_puzzle` model support * 07/07/2026 7.3.0-dev `main`: ✨ Added `deepseek_vl` model support @@ -258,24 +259,24 @@ Selected public references where teams or companies explicitly mention GPT-QMode ## Model Support -| Model | | | | | | | | | | -|-------------------------------|---|---------------------------------|--|------------------|--|---------------------------------|--|------------------------|---| -| Apertus | ✅ | EXAONE 3/4 | ✅ | Dots1 | ✅ | Mistral3 / Ministral3 | ✅ | Qwen 2/3/3.5 (Next/MoE) | ✅ | -| Baichuan | ✅ | Falcon (H1 / Mamba) | ✅ | InternLM 1/2/2.5 | ✅ | Mixtral | ✅ | Qwen 2/2.5/3 VL | ✅ | -| Bloom | ✅ | FastVLM | ✅ | Kimi K2 | ✅ | MobileLLM | ✅ | Qwen 2.5/3 Omni | ✅ | -| ChatGLM | ✅ | Gemma 1-4 / 3n | ✅ | Klear | ✅ | MOSS | ✅ | RefinedWeb | ✅ | -| CodeGen | ✅ | GPTBigCode | ✅ | LING/RING | ✅ | MPT | ✅ | StableLM | ✅ | -| Cohere 1-2 / 2 MoE | ✅ | GPT-Neo / NeoX | ✅ | Llama 1-3.3 | ✅ | Nemotron H / H Puzzle / Omni | ✅ | StarCoder2 | ✅ | -| DBRX Converted | ✅ | GPT-2 | ✅ | Llama 3.2 VL | ✅ | Nemotron Ultra / Labs-Diffusion | ✅ | TeleChat2 | ✅ | -| Deci | ✅ | GPT-J | ✅ | Llama 4 | ✅ | OPT | ✅ | Trinity | ✅ | -| DeepSeek-V2/V3/V4/R1 | ✅ | GPT-OSS | ✅ | LongCat Flash | ✅ | OLMo2 / LLaDA2 | ✅ | Yi | ✅ | -| DeepSeek-V2 Lite / VL / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ | -| Dream | ✅ | GRIN-MoE | ✅ | Instella | ✅ | Phi 1-4 | ✅ | Voxtral | ✅ | +| Model | | | | | | | | | | +|-------------------------------|---|---------------------------------|--|----------------------------|--|---------------------------------|--|------------------------|---| +| Apertus | ✅ | EXAONE 3/4 | ✅ | Dots1 | ✅ | Mistral3 / Ministral3 | ✅ | Qwen 2/3/3.5 (Next/MoE) | ✅ | +| Baichuan | ✅ | Falcon (H1 / Mamba) | ✅ | InternLM 1/2/2.5 | ✅ | Mixtral | ✅ | Qwen 2/2.5/3 VL | ✅ | +| Bloom | ✅ | FastVLM | ✅ | Kimi K2 | ✅ | MobileLLM | ✅ | Qwen 2.5/3 Omni | ✅ | +| ChatGLM | ✅ | Gemma 1-4 / 3n | ✅ | Klear | ✅ | MOSS | ✅ | RefinedWeb | ✅ | +| CodeGen | ✅ | GPTBigCode | ✅ | LING/RING | ✅ | MPT | ✅ | StableLM | ✅ | +| Cohere 1-2 / 2 MoE | ✅ | GPT-Neo / NeoX | ✅ | Llama 1-3.3 | ✅ | Nemotron H / H Puzzle / Omni | ✅ | StarCoder2 | ✅ | +| DBRX Converted | ✅ | GPT-2 | ✅ | Llama 3.2 VL | ✅ | Nemotron Ultra / Labs-Diffusion | ✅ | TeleChat2 | ✅ | +| Deci | ✅ | GPT-J | ✅ | Llama 4 | ✅ | OPT | ✅ | Trinity | ✅ | +| DeepSeek-V2/V3/V4/R1 | ✅ | GPT-OSS | ✅ | LongCat Flash | ✅ | OLMo2 / LLaDA2 | ✅ | Yi | ✅ | +| DeepSeek-V2 Lite / VL / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ | +| Dream | ✅ | GRIN-MoE | ✅ | Instella | ✅ | Phi 1-4 | ✅ | Voxtral | ✅ | | ERNIE 4.5 / MoE / VL MoE | ✅ | GLM 4/4V/4.5V/4.6V/5/5.1/OCR/ASR | ✅ | GLM4 MoE / Lite / 4.5V MoE | ✅ | MiniCPM 3/O/V/V 4_6 | ✅ | PanGu-α | ✅ | -| XVERSE | ✅ | Brumby | ✅ | Hymba | ✅ | Mistral | ✅ | Qwen 1/2/3/3.5 | ✅ | -| MiniMax M2/M3 | ✅ | AfMoE | ✅ | Bailing-MoE | ✅ | LFM2 / LFM2-VL / LFM2-MoE | ✅ | Marin | ✅ | -| InternVL Chat | ✅ | Laguna | ✅ | Mimo / Mimo V2 | ✅ | Zamba / Zamba2 | ✅ | Intern S1 | ✅ | -| HunYuan V1 Dense / MoE | ✅ | HY-V3 | ✅ | | | | | | | +| XVERSE | ✅ | Brumby | ✅ | Hymba | ✅ | Mistral | ✅ | Qwen 1/2/3/3.5 | ✅ | +| MiniMax M2/M3 | ✅ | AfMoE | ✅ | Bailing-MoE | ✅ | LFM2 / LFM2-VL / LFM2-MoE | ✅ | Marin | ✅ | +| InternVL Chat | ✅ | Laguna | ✅ | Mimo / Mimo V2 | ✅ | Zamba / Zamba2 | ✅ | Intern S1 | ✅ | +| HunYuan V1 Dense / MoE | ✅ | HY-V3 | ✅ | Inkling | ✅ | | | | | Prism Bonsai GGUF checkpoints are supported for inference only through GPT-QModel's native GGUF path and internal GGUF runtime. Bonsai checkpoints load through the normal model path or repo argument and do not require the external `gguf` package. Prism model quantization is not included.