Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

## Latest News

* 07/07/2026 7.1.0-dev `main`: ✨ Added `deepseek_vl` model support
* 07/06/2026 7.1.0-dev `main`: ✨ Added `deepseek_ocr2` model support
* 04/07/2026 7.1.0-dev `main`: ✨ Added `deepseek_vl_v2` model support
* 02/07/2026 7.1.0-dev `main`: ✨ Added `lfm2` and `lfm2_vl` model support
Expand Down Expand Up @@ -266,7 +267,7 @@ Selected public references where teams or companies explicitly mention GPT-QMode
| DBRX Converted | ✅ | GPT-2 | ✅ | Llama 3.2 VL | ✅ | Nemotron Ultra / Labs-Diffusion | ✅ | TeleChat2 | ✅ |
| Deci | ✅ | GPT-J | ✅ | Llama 4 | ✅ | OPT | ✅ | Trinity | ✅ |
| DeepSeek-V2/V3/V4/R1 | ✅ | GPT-OSS | ✅ | LongCat Flash | ✅ | OLMo2 / LLaDA2 | ✅ | Yi | ✅ |
| DeepSeek-V2 Lite / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ |
| DeepSeek-V2 Lite / VL / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ |
| Dream | ✅ | GRIN-MoE | ✅ | Instella | ✅ | Phi 1-4 | ✅ | Voxtral | ✅ |
| ERNIE 4.5 / MoE / VL MoE | ✅ | GLM 4/4V/4.5V/4.6V/5/5.1/OCR/ASR | ✅ | GLM4 MoE / Lite / 4.5V MoE | ✅ | MiniCPM 3/O/V/V 4_6 | ✅ | PanGu-α | ✅ |
| XVERSE | ✅ | Brumby | ✅ | Hymba | ✅ | Mistral | ✅ | Qwen 1/2/3/3.5 | ✅ |
Expand Down
2 changes: 2 additions & 0 deletions gptqmodel/models/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
from .definitions.deepseek_v3 import DeepSeekV3QModel # noqa: E402
from .definitions.deepseek_v4 import DeepSeekV4QModel # noqa: E402
from .definitions.deepseek_ocr2 import DeepSeekOCR2QModel # noqa: E402
from .definitions.deepseek_vl import DeepSeekVLQModel # noqa: E402
from .definitions.deepseek_vl_v2 import DeepSeekVLV2QModel # noqa: E402
from .definitions.dots1 import Dots1QModel # noqa: E402
from .definitions.dream import DreamQModel # noqa: E402
Expand Down Expand Up @@ -301,6 +302,7 @@
"deepseek_v3": DeepSeekV3QModel,
"deepseek_v4": DeepSeekV4QModel,
"deepseek_ocr2": DeepSeekOCR2QModel,
"deepseek_vl": DeepSeekVLQModel,
"deepseek_vl_v2": DeepSeekVLV2QModel,
"dots1": Dots1QModel,
"exaone": ExaOneQModel,
Expand Down
1 change: 1 addition & 0 deletions gptqmodel/models/definitions/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from .deepseek_v3 import DeepSeekV3QModel
from .deepseek_v4 import DeepSeekV4QModel
from .deepseek_ocr2 import DeepSeekOCR2QModel
from .deepseek_vl import DeepSeekVLQModel
from .deepseek_vl_v2 import DeepSeekVLV2QModel
from .dots1 import Dots1QModel
from .dream import DreamQModel
Expand Down
144 changes: 144 additions & 0 deletions gptqmodel/models/definitions/deepseek_vl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
# SPDX-FileCopyrightText: 2026 ModelCloud.ai
# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai
# SPDX-License-Identifier: Apache-2.0
# Contact: qubitium@modelcloud.ai, x.com/qubitium

from types import SimpleNamespace
from typing import Any, Dict

import torch
from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin

from ...utils.calibration import batched
from ...utils.model import MODALITY, move_to
from ...utils.offload import offload_to_disk
from .._const import CPU
from ..base import BaseQModel


class DeepSeekVLQModel(BaseQModel):
loader = AutoModelForImageTextToText
require_load_processor = True
support_batch_quantize = False

modality = [MODALITY.TEXT, MODALITY.IMAGE_TO_TEXT]

pre_lm_head_norm_module = "model.language_model.norm"
rotary_embedding = "model.language_model.rotary_emb"
# layer_modules_strict = False

# HF_CONVERSION_MAP_REVERSED = (
# # DeepSeek-VL mounts `SiglipVisionModel` at `model.vision_model`.
# # Runtime shell paths expose `model.vision_model.*`, while the original
# # checkpoint keeps SigLIP's base prefix under `model.vision_model.vision_model.*`.
# SimpleNamespace(
# source_patterns=[r"^model\.vision_model\.(?!vision_model\.)(.+)$"],
# target_patterns=[r"^model.vision_model.vision_model.\1"],
# operations=[],
# ),
# )

module_tree = [
"model",
"language_model",
"layers",
"#",
{
"input_layernorm": ("input_layernorm:!",),
"self_attn": ("q_proj:0", "k_proj:0", "v_proj:0", "o_proj:1"),
"post_attention_layernorm": ("post_attention_layernorm:!",),
"mlp": ("gate_proj:0", "up_proj:0", "down_proj:1"),
},
]

def pre_quantize_generate_hook_start(self):
core_model = self.model.model
language_model = core_model.language_model
self.shell_module_materialize(language_model.embed_tokens, self.quantize_config.device)
self.shell_module_materialize(language_model.norm, self.quantize_config.device)
self.shell_module_materialize(language_model.rotary_emb, self.quantize_config.device)
self.shell_module_materialize(core_model.vision_model, self.quantize_config.device)
self.shell_module_materialize(core_model.aligner, self.quantize_config.device)

def pre_quantize_generate_hook_end(self):
core_model = self.model.model
language_model = core_model.language_model
if self.quantize_config.offload_to_disk:
offload_to_disk(
model=language_model,
module=language_model.embed_tokens,
disk_path=self.quantize_config.offload_to_disk_path,
)
offload_to_disk(
model=language_model,
module=language_model.norm,
disk_path=self.quantize_config.offload_to_disk_path,
)
offload_to_disk(
model=language_model,
module=language_model.rotary_emb,
disk_path=self.quantize_config.offload_to_disk_path,
)
offload_to_disk(
model=core_model,
module=core_model.vision_model,
disk_path=self.quantize_config.offload_to_disk_path,
)
offload_to_disk(
model=core_model,
module=core_model.aligner,
disk_path=self.quantize_config.offload_to_disk_path,
)
return

language_model.embed_tokens = move_to(language_model.embed_tokens, device=CPU)
language_model.norm = move_to(language_model.norm, device=CPU)
language_model.rotary_emb = move_to(language_model.rotary_emb, device=CPU)
core_model.vision_model = move_to(core_model.vision_model, device=CPU)
core_model.aligner = move_to(core_model.aligner, device=CPU)

def preprocess_dataset(self, sample: Dict) -> Dict:
return sample

def load_processor(self) -> ProcessorMixin:
return AutoProcessor.from_pretrained(self.model_local_path, trust_remote_code=False)

@classmethod
def prepare_inputs_for_conversations(
cls,
processor: ProcessorMixin,
conversations: list[dict] | list[list[dict]],
):
return processor.apply_chat_template(
conversations,
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
)

def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
del kwargs
processor = self.load_processor()
calib_data = []
for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
calib_data.append(self.prepare_inputs_for_conversations(processor, batch))
del processor
return calib_data

def move_input_capture_example(self, example: Dict[str, Any], data_device: torch.device) -> Dict[str, Any]:
example = super().move_input_capture_example(example, data_device)
pixel_values = example.get("pixel_values")
if torch.is_tensor(pixel_values):
vision_model = self.model.model.vision_model
first_parameter = next(vision_model.parameters(), None)
vision_device = getattr(first_parameter, "device", pixel_values.device)
vision_dtype = getattr(vision_model, "dtype", None)
if not isinstance(vision_dtype, torch.dtype):
vision_dtype = getattr(first_parameter, "dtype", None)
if isinstance(vision_dtype, torch.dtype):
example["pixel_values"] = pixel_values.to(device=vision_device, dtype=vision_dtype)
return example


__all__ = ["DeepSeekVLQModel"]
26 changes: 26 additions & 0 deletions tests/models/ovis/image_to_test_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from gptqmodel.models.definitions.base_qwen2_5_omni import BaseQwen2_5_OmniGPTQ
from gptqmodel.models.definitions.base_qwen2_vl import BaseQwen2VLGPTQ
from gptqmodel.models.definitions.deepseek_ocr2 import DeepSeekOCR2QModel
from gptqmodel.models.definitions.deepseek_vl import DeepSeekVLQModel
from gptqmodel.models.definitions.deepseek_vl_v2 import DeepSeekVLV2QModel
from gptqmodel.models.definitions.ernie4_5_vl_moe import Ernie4_5_VLMoeQModel
from gptqmodel.models.definitions.interns1 import InternS1QModel
Expand Down Expand Up @@ -72,6 +73,24 @@ def format_deepseek_vl_v2_dataset(image, assistant):
]


def format_deepseek_vl_dataset(image, assistant):
return [
{
"role": "user",
"content": [
{"type": "image", "url": image},
{"type": "text", "text": "generate a caption for this image"},
],
},
{
"role": "assistant",
"content": [
{"type": "text", "text": assistant},
],
},
]


def format_deepseek_ocr2_dataset(image, assistant):
del assistant
return {
Expand Down Expand Up @@ -116,6 +135,10 @@ def prepare_deepseek_vl_v2_dataset(n_sample: int = 20) -> list[list[dict]]:
return prepare_dataset(format_deepseek_vl_v2_dataset, n_sample=n_sample)


def prepare_deepseek_vl_dataset(n_sample: int = 20) -> list[list[dict]]:
return prepare_dataset(format_deepseek_vl_dataset, n_sample=n_sample)


def prepare_deepseek_ocr2_dataset(n_sample: int = 20) -> list[dict]:
return prepare_dataset(format_deepseek_ocr2_dataset, n_sample=n_sample)

Expand Down Expand Up @@ -152,6 +175,9 @@ def get_calib_dataset(model):
if isinstance(model, DeepSeekVLV2QModel):
return prepare_deepseek_vl_v2_dataset(n_sample=20)

if isinstance(model, DeepSeekVLQModel):
return prepare_deepseek_vl_dataset(n_sample=20)

if isinstance(model, DeepSeekOCR2QModel):
return prepare_deepseek_ocr2_dataset(n_sample=20)

Expand Down
Loading