diff --git a/README.md b/README.md index 03b2c6589..c611ab031 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ ## Latest News +* 07/07/2026 7.1.0-dev `main`: ✨ Added `deepseek_vl` model support * 07/06/2026 7.1.0-dev `main`: ✨ Added `deepseek_ocr2` model support * 04/07/2026 7.1.0-dev `main`: ✨ Added `deepseek_vl_v2` model support * 02/07/2026 7.1.0-dev `main`: ✨ Added `lfm2` and `lfm2_vl` model support @@ -266,7 +267,7 @@ Selected public references where teams or companies explicitly mention GPT-QMode | DBRX Converted | ✅ | GPT-2 | ✅ | Llama 3.2 VL | ✅ | Nemotron Ultra / Labs-Diffusion | ✅ | TeleChat2 | ✅ | | Deci | ✅ | GPT-J | ✅ | Llama 4 | ✅ | OPT | ✅ | Trinity | ✅ | | DeepSeek-V2/V3/V4/R1 | ✅ | GPT-OSS | ✅ | LongCat Flash | ✅ | OLMo2 / LLaDA2 | ✅ | Yi | ✅ | -| DeepSeek-V2 Lite / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ | +| DeepSeek-V2 Lite / VL / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ | | Dream | ✅ | GRIN-MoE | ✅ | Instella | ✅ | Phi 1-4 | ✅ | Voxtral | ✅ | | ERNIE 4.5 / MoE / VL MoE | ✅ | GLM 4/4V/4.5V/4.6V/5/5.1/OCR/ASR | ✅ | GLM4 MoE / Lite / 4.5V MoE | ✅ | MiniCPM 3/O/V/V 4_6 | ✅ | PanGu-α | ✅ | | XVERSE | ✅ | Brumby | ✅ | Hymba | ✅ | Mistral | ✅ | Qwen 1/2/3/3.5 | ✅ | diff --git a/gptqmodel/models/auto.py b/gptqmodel/models/auto.py index 14f95a3d8..5198ca30e 100644 --- a/gptqmodel/models/auto.py +++ b/gptqmodel/models/auto.py @@ -87,6 +87,7 @@ from .definitions.deepseek_v3 import DeepSeekV3QModel # noqa: E402 from .definitions.deepseek_v4 import DeepSeekV4QModel # noqa: E402 from .definitions.deepseek_ocr2 import DeepSeekOCR2QModel # noqa: E402 +from .definitions.deepseek_vl import DeepSeekVLQModel # noqa: E402 from .definitions.deepseek_vl_v2 import DeepSeekVLV2QModel # noqa: E402 from .definitions.dots1 import Dots1QModel # noqa: E402 from .definitions.dream import DreamQModel # noqa: E402 @@ -301,6 +302,7 @@ "deepseek_v3": DeepSeekV3QModel, "deepseek_v4": DeepSeekV4QModel, "deepseek_ocr2": DeepSeekOCR2QModel, + "deepseek_vl": DeepSeekVLQModel, "deepseek_vl_v2": DeepSeekVLV2QModel, "dots1": Dots1QModel, "exaone": ExaOneQModel, diff --git a/gptqmodel/models/definitions/__init__.py b/gptqmodel/models/definitions/__init__.py index b32d8e715..8dbba5b5f 100644 --- a/gptqmodel/models/definitions/__init__.py +++ b/gptqmodel/models/definitions/__init__.py @@ -21,6 +21,7 @@ from .deepseek_v3 import DeepSeekV3QModel from .deepseek_v4 import DeepSeekV4QModel from .deepseek_ocr2 import DeepSeekOCR2QModel +from .deepseek_vl import DeepSeekVLQModel from .deepseek_vl_v2 import DeepSeekVLV2QModel from .dots1 import Dots1QModel from .dream import DreamQModel diff --git a/gptqmodel/models/definitions/deepseek_vl.py b/gptqmodel/models/definitions/deepseek_vl.py new file mode 100644 index 000000000..657dcc2a7 --- /dev/null +++ b/gptqmodel/models/definitions/deepseek_vl.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from types import SimpleNamespace +from typing import Any, Dict + +import torch +from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin + +from ...utils.calibration import batched +from ...utils.model import MODALITY, move_to +from ...utils.offload import offload_to_disk +from .._const import CPU +from ..base import BaseQModel + + +class DeepSeekVLQModel(BaseQModel): + loader = AutoModelForImageTextToText + require_load_processor = True + support_batch_quantize = False + + modality = [MODALITY.TEXT, MODALITY.IMAGE_TO_TEXT] + + pre_lm_head_norm_module = "model.language_model.norm" + rotary_embedding = "model.language_model.rotary_emb" + # layer_modules_strict = False + + # HF_CONVERSION_MAP_REVERSED = ( + # # DeepSeek-VL mounts `SiglipVisionModel` at `model.vision_model`. + # # Runtime shell paths expose `model.vision_model.*`, while the original + # # checkpoint keeps SigLIP's base prefix under `model.vision_model.vision_model.*`. + # SimpleNamespace( + # source_patterns=[r"^model\.vision_model\.(?!vision_model\.)(.+)$"], + # target_patterns=[r"^model.vision_model.vision_model.\1"], + # operations=[], + # ), + # ) + + module_tree = [ + "model", + "language_model", + "layers", + "#", + { + "input_layernorm": ("input_layernorm:!",), + "self_attn": ("q_proj:0", "k_proj:0", "v_proj:0", "o_proj:1"), + "post_attention_layernorm": ("post_attention_layernorm:!",), + "mlp": ("gate_proj:0", "up_proj:0", "down_proj:1"), + }, + ] + + def pre_quantize_generate_hook_start(self): + core_model = self.model.model + language_model = core_model.language_model + self.shell_module_materialize(language_model.embed_tokens, self.quantize_config.device) + self.shell_module_materialize(language_model.norm, self.quantize_config.device) + self.shell_module_materialize(language_model.rotary_emb, self.quantize_config.device) + self.shell_module_materialize(core_model.vision_model, self.quantize_config.device) + self.shell_module_materialize(core_model.aligner, self.quantize_config.device) + + def pre_quantize_generate_hook_end(self): + core_model = self.model.model + language_model = core_model.language_model + if self.quantize_config.offload_to_disk: + offload_to_disk( + model=language_model, + module=language_model.embed_tokens, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=language_model, + module=language_model.norm, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=language_model, + module=language_model.rotary_emb, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=core_model, + module=core_model.vision_model, + disk_path=self.quantize_config.offload_to_disk_path, + ) + offload_to_disk( + model=core_model, + module=core_model.aligner, + disk_path=self.quantize_config.offload_to_disk_path, + ) + return + + language_model.embed_tokens = move_to(language_model.embed_tokens, device=CPU) + language_model.norm = move_to(language_model.norm, device=CPU) + language_model.rotary_emb = move_to(language_model.rotary_emb, device=CPU) + core_model.vision_model = move_to(core_model.vision_model, device=CPU) + core_model.aligner = move_to(core_model.aligner, device=CPU) + + def preprocess_dataset(self, sample: Dict) -> Dict: + return sample + + def load_processor(self) -> ProcessorMixin: + return AutoProcessor.from_pretrained(self.model_local_path, trust_remote_code=False) + + @classmethod + def prepare_inputs_for_conversations( + cls, + processor: ProcessorMixin, + conversations: list[dict] | list[list[dict]], + ): + return processor.apply_chat_template( + conversations, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ) + + def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): + del kwargs + processor = self.load_processor() + calib_data = [] + for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + calib_data.append(self.prepare_inputs_for_conversations(processor, batch)) + del processor + return calib_data + + def move_input_capture_example(self, example: Dict[str, Any], data_device: torch.device) -> Dict[str, Any]: + example = super().move_input_capture_example(example, data_device) + pixel_values = example.get("pixel_values") + if torch.is_tensor(pixel_values): + vision_model = self.model.model.vision_model + first_parameter = next(vision_model.parameters(), None) + vision_device = getattr(first_parameter, "device", pixel_values.device) + vision_dtype = getattr(vision_model, "dtype", None) + if not isinstance(vision_dtype, torch.dtype): + vision_dtype = getattr(first_parameter, "dtype", None) + if isinstance(vision_dtype, torch.dtype): + example["pixel_values"] = pixel_values.to(device=vision_device, dtype=vision_dtype) + return example + + +__all__ = ["DeepSeekVLQModel"] diff --git a/tests/models/ovis/image_to_test_dataset.py b/tests/models/ovis/image_to_test_dataset.py index dcf875829..f7ed51ec4 100644 --- a/tests/models/ovis/image_to_test_dataset.py +++ b/tests/models/ovis/image_to_test_dataset.py @@ -5,6 +5,7 @@ from gptqmodel.models.definitions.base_qwen2_5_omni import BaseQwen2_5_OmniGPTQ from gptqmodel.models.definitions.base_qwen2_vl import BaseQwen2VLGPTQ from gptqmodel.models.definitions.deepseek_ocr2 import DeepSeekOCR2QModel +from gptqmodel.models.definitions.deepseek_vl import DeepSeekVLQModel from gptqmodel.models.definitions.deepseek_vl_v2 import DeepSeekVLV2QModel from gptqmodel.models.definitions.ernie4_5_vl_moe import Ernie4_5_VLMoeQModel from gptqmodel.models.definitions.interns1 import InternS1QModel @@ -72,6 +73,24 @@ def format_deepseek_vl_v2_dataset(image, assistant): ] +def format_deepseek_vl_dataset(image, assistant): + return [ + { + "role": "user", + "content": [ + {"type": "image", "url": image}, + {"type": "text", "text": "generate a caption for this image"}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": assistant}, + ], + }, + ] + + def format_deepseek_ocr2_dataset(image, assistant): del assistant return { @@ -116,6 +135,10 @@ def prepare_deepseek_vl_v2_dataset(n_sample: int = 20) -> list[list[dict]]: return prepare_dataset(format_deepseek_vl_v2_dataset, n_sample=n_sample) +def prepare_deepseek_vl_dataset(n_sample: int = 20) -> list[list[dict]]: + return prepare_dataset(format_deepseek_vl_dataset, n_sample=n_sample) + + def prepare_deepseek_ocr2_dataset(n_sample: int = 20) -> list[dict]: return prepare_dataset(format_deepseek_ocr2_dataset, n_sample=n_sample) @@ -152,6 +175,9 @@ def get_calib_dataset(model): if isinstance(model, DeepSeekVLV2QModel): return prepare_deepseek_vl_v2_dataset(n_sample=20) + if isinstance(model, DeepSeekVLQModel): + return prepare_deepseek_vl_dataset(n_sample=20) + if isinstance(model, DeepSeekOCR2QModel): return prepare_deepseek_ocr2_dataset(n_sample=20) diff --git a/tests/models/test_deepseek_vl.py b/tests/models/test_deepseek_vl.py new file mode 100644 index 000000000..ec8d5f862 --- /dev/null +++ b/tests/models/test_deepseek_vl.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +import os.path +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from model_test import ModelTest +from ovis import image_to_test_dataset +from PIL import Image +from safetensors.torch import save_file +from torch import nn +from transformers import AutoConfig, AutoModelForImageTextToText + +from gptqmodel.models import auto +from gptqmodel.models.definitions.deepseek_vl import DeepSeekVLQModel +from gptqmodel.utils.structure import LazyTurtle + + +MODEL_PATH = Path("/monster/data/model/deepseek-vl-1.3b-chat") + + +def test_deepseek_vl_model_type_selects_definition(monkeypatch): + fake_config = SimpleNamespace(model_type="deepseek_vl") + + monkeypatch.setattr(auto, "resolve_trust_remote_code", lambda path, trust_remote_code=False: trust_remote_code) + monkeypatch.setattr(auto.AutoConfig, "from_pretrained", lambda *args, **kwargs: fake_config) + + assert auto.check_and_get_model_definition("/tmp/deepseek-vl") is DeepSeekVLQModel + + +def test_deepseek_vl_module_tree_covers_llama_decoder_paths(): + layer_modules = DeepSeekVLQModel.simple_layer_modules( + model_config=SimpleNamespace(), + quantize_config=SimpleNamespace(dynamic=None), + ) + flat_modules = {name for block in layer_modules for name in block} + + assert DeepSeekVLQModel.layer_modules_strict is False + assert DeepSeekVLQModel.require_load_processor is True + assert DeepSeekVLQModel.support_batch_quantize is False + assert DeepSeekVLQModel.pre_lm_head_norm_module == "model.language_model.norm" + assert DeepSeekVLQModel.rotary_embedding == "model.language_model.rotary_emb" + assert DeepSeekVLQModel.extract_layers_node() == ["model.language_model.layers"] + + assert "self_attn.q_proj" in flat_modules + assert "self_attn.k_proj" in flat_modules + assert "self_attn.v_proj" in flat_modules + assert "self_attn.o_proj" in flat_modules + assert "mlp.gate_proj" in flat_modules + assert "mlp.up_proj" in flat_modules + assert "mlp.down_proj" in flat_modules + + +def test_deepseek_vl_base_modules_include_vision_aligner_and_language_roots(): + class _LanguageModel(nn.Module): + def __init__(self): + super().__init__() + self.embed_tokens = nn.Embedding(4, 4) + self.layers = nn.ModuleList([nn.Identity()]) + self.norm = nn.LayerNorm(4) + self.rotary_emb = nn.Identity() + + class _CoreModel(nn.Module): + def __init__(self): + super().__init__() + self.vision_model = nn.Identity() + self.aligner = nn.Identity() + self.language_model = _LanguageModel() + + class _Wrapper(nn.Module): + def __init__(self): + super().__init__() + self.model = _CoreModel() + + base_modules = set(DeepSeekVLQModel.get_base_modules(_Wrapper())) + + assert "model.vision_model" in base_modules + assert "model.aligner" in base_modules + assert "model.language_model.embed_tokens" in base_modules + assert "model.language_model.norm" in base_modules + assert "model.language_model.rotary_emb" in base_modules + + +def test_deepseek_vl_pre_quantize_hook_materializes_base_modules(): + model = nn.Module() + model.model = nn.Module() + model.model.language_model = nn.Module() + model.model.language_model.embed_tokens = nn.Embedding(4, 4) + model.model.language_model.norm = nn.LayerNorm(4) + model.model.language_model.rotary_emb = nn.Identity() + model.model.vision_model = nn.Linear(4, 4) + model.model.aligner = nn.Linear(4, 4) + + qmodel = object.__new__(DeepSeekVLQModel) + nn.Module.__init__(qmodel) + qmodel.model = model + qmodel.quantize_config = SimpleNamespace(device=torch.device("cpu")) + materialized = [] + + def shell_module_materialize(module, device): + materialized.append((module, device)) + return module + + qmodel.shell_module_materialize = shell_module_materialize + + qmodel.pre_quantize_generate_hook_start() + + assert materialized == [ + (model.model.language_model.embed_tokens, torch.device("cpu")), + (model.model.language_model.norm, torch.device("cpu")), + (model.model.language_model.rotary_emb, torch.device("cpu")), + (model.model.vision_model, torch.device("cpu")), + (model.model.aligner, torch.device("cpu")), + ] + + +def test_prepare_deepseek_vl_dataset_reuses_shared_dataset(monkeypatch): + calls = {} + + def fake_prepare_dataset(format_func, n_sample): + calls["format_func"] = format_func + calls["n_sample"] = n_sample + return [format_func("https://example.com/cat.jpg", "caption")] + + monkeypatch.setattr(image_to_test_dataset, "prepare_dataset", fake_prepare_dataset) + + dataset = image_to_test_dataset.prepare_deepseek_vl_dataset(n_sample=3) + + assert calls == { + "format_func": image_to_test_dataset.format_deepseek_vl_dataset, + "n_sample": 3, + } + assert dataset == [ + [ + { + "role": "user", + "content": [ + {"type": "image", "url": "https://example.com/cat.jpg"}, + {"type": "text", "text": "generate a caption for this image"}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "caption"}, + ], + }, + ] + ] + + +def test_deepseek_vl_lazy_turtle_resolves_siglip_vision_model_prefix(tmp_path): + checkpoint_tensors = { + "model.vision_model.vision_model.embeddings.patch_embedding.weight": torch.zeros(2, 2), + } + model_dir = tmp_path / "source_model" + model_dir.mkdir() + shard_name = "model.safetensors" + save_file(checkpoint_tensors, str(model_dir / shard_name)) + (model_dir / "model.safetensors.index.json").write_text( + '{"weight_map":{"model.vision_model.vision_model.embeddings.patch_embedding.weight":"model.safetensors"}}', + encoding="utf-8", + ) + + class _PatchEmbedding(nn.Module): + def __init__(self): + super().__init__() + self.weight = nn.Parameter(torch.empty(2, 2, device="meta")) + + class _Embeddings(nn.Module): + def __init__(self): + super().__init__() + self.patch_embedding = _PatchEmbedding() + + class _VisionModel(nn.Module): + def __init__(self): + super().__init__() + self.embeddings = _Embeddings() + + class _CoreModel(nn.Module): + def __init__(self): + super().__init__() + self.vision_model = _VisionModel() + + class _Wrapper(nn.Module): + def __init__(self): + super().__init__() + self.model = _CoreModel() + + shell = _Wrapper() + turtle = LazyTurtle.maybe_create( + model_local_path=str(model_dir), + config=SimpleNamespace(_experts_implementation=None), + model_init_kwargs={"device_map": {"": "cpu"}}, + module_tree=DeepSeekVLQModel.module_tree, + hf_conversion_map_reversed=DeepSeekVLQModel.resolve_hf_conversion_map_reversed(), + target_model=shell, + ) + + assert turtle is not None + assert ( + turtle._resolve_checkpoint_tensor_name( + "model.vision_model.embeddings.patch_embedding", + "weight", + ) + == "model.vision_model.vision_model.embeddings.patch_embedding.weight" + ) + + +@pytest.mark.skipif(not MODEL_PATH.exists(), reason="DeepSeek-VL model not found") +def test_deepseek_vl_native_shell_matches_definition_tree(): + from accelerate import init_empty_weights + + config = AutoConfig.from_pretrained(MODEL_PATH, trust_remote_code=False) + with init_empty_weights(include_buffers=True): + shell = AutoModelForImageTextToText.from_config(config, trust_remote_code=False) + + layer = shell.model.language_model.layers[0] + + assert config.model_type == "deepseek_vl" + assert auto.check_and_get_model_definition(MODEL_PATH) is DeepSeekVLQModel + assert hasattr(shell.model, "vision_model") + assert hasattr(shell.model, "aligner") + assert hasattr(shell.model, "language_model") + assert hasattr(layer.self_attn, "q_proj") + assert hasattr(layer.self_attn, "k_proj") + assert hasattr(layer.self_attn, "v_proj") + assert hasattr(layer.self_attn, "o_proj") + assert hasattr(layer.mlp, "gate_proj") + assert hasattr(layer.mlp, "up_proj") + assert hasattr(layer.mlp, "down_proj") + + +class TestDeepSeekVL(ModelTest): + NATIVE_MODEL_ID = "/monster/data/model/deepseek-vl-1.3b-chat" + TRUST_REMOTE_CODE = False + USE_FLASH_ATTN = False + OFFLOAD_TO_DISK = False + EVAL_BATCH_SIZE = 1 + + def test_deepseek_vl(self): + with self.model_compat_test_context(): + model, _tokenizer, processor = self.quantModel( + self.NATIVE_MODEL_ID, + trust_remote_code=self.TRUST_REMOTE_CODE, + dtype=self.TORCH_DTYPE, + batch_size=1, + call_perform_post_quant_validation=False, + ) + + image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ovis/10016.jpg") + image = Image.open(image_path).convert("RGB") + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"}, + {"type": "text", "text": "What does this picture show?"}, + ], + }, + ] + + inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ).to(model.device, dtype=model.dtype) + + generated_ids = model.generate(**inputs, max_new_tokens=128) + print("ggg", generated_ids) + generated_ids_trimmed = [ + out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) + ] + output_text = processor.batch_decode( + generated_ids_trimmed, + skip_special_tokens=True, + clean_up_tokenization_spaces=False, + )[0] + print("output_text:", output_text) + + self.assertIn("snow", output_text.lower()) + self.check_kernel(model, self.KERNEL_INFERENCE)