diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 00000000..b58b603f
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,5 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
diff --git a/.idea/copilot.data.migration.agent.xml b/.idea/copilot.data.migration.agent.xml
new file mode 100644
index 00000000..4ea72a91
--- /dev/null
+++ b/.idea/copilot.data.migration.agent.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AgentMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 00000000..105ce2da
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 00000000..35aab3dd
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="piton311" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="hf_312" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/mlx-vlm.iml b/.idea/mlx-vlm.iml
new file mode 100644
index 00000000..8d3acc44
--- /dev/null
+++ b/.idea/mlx-vlm.iml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="hf_312" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="GOOGLE" />
+    <option name="myDocStringFormat" value="Google" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="py.test" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 00000000..04e48ab4
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/mlx-vlm.iml" filepath="$PROJECT_DIR$/.idea/mlx-vlm.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 00000000..35eb1ddf
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/mlx_vlm/models/colqwen3/__init__.py b/mlx_vlm/models/colqwen3/__init__.py
new file mode 100644
index 00000000..7813bf31
--- /dev/null
+++ b/mlx_vlm/models/colqwen3/__init__.py
@@ -0,0 +1,15 @@
+from .colqwen3 import Model
+from .config import ModelConfig, TextConfig, VisionConfig
+
+# utils.py bunları arıyor:
+from .vision import VisionModel
+from .language import LanguageModel
+
+__all__ = [
+    "Model",
+    "ModelConfig",
+    "TextConfig",
+    "VisionConfig",
+    "VisionModel",
+    "LanguageModel",
+]
diff --git a/mlx_vlm/models/colqwen3/colqwen3.py b/mlx_vlm/models/colqwen3/colqwen3.py
new file mode 100644
index 00000000..b3f8d9a4
--- /dev/null
+++ b/mlx_vlm/models/colqwen3/colqwen3.py
@@ -0,0 +1,473 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any, Optional, Dict, List
+
+import mlx.core as mx
+import mlx.nn as nn
+import numpy as np
+
+from .config import ModelConfig
+from .language import LanguageModel
+from .vision import VisionModel
+
+
+# -----------------------------------------------------------------------------
+# Robust converters (torch / numpy / list -> mx.array)
+# -----------------------------------------------------------------------------
+def _as_np(x: Any) -> Optional[np.ndarray]:
+    if x is None:
+        return None
+    if isinstance(x, np.ndarray):
+        return x
+    if isinstance(x, mx.array):
+        return np.array(x)
+
+    # torch.Tensor -> numpy
+    try:
+        import torch  # optional
+        if isinstance(x, torch.Tensor):
+            return x.detach().cpu().numpy()
+    except Exception:
+        pass
+
+    # generic ".numpy()"
+    if hasattr(x, "numpy") and callable(x.numpy):
+        try:
+            return x.numpy()
+        except Exception:
+            pass
+
+    # list/tuple/scalar
+    try:
+        return np.asarray(x)
+    except Exception:
+        return None
+
+
+def _as_mx(x: Any) -> Any:
+    """
+    Convert to mx.array when possible; otherwise return as-is (e.g. None).
+    """
+    if x is None:
+        return None
+    if isinstance(x, mx.array):
+        return x
+    arr = _as_np(x)
+    if arr is None:
+        return x
+    return mx.array(arr)
+
+
+def _as_mx_int32(x: Any) -> Optional[mx.array]:
+    if x is None:
+        return None
+    if isinstance(x, mx.array):
+        return x if x.dtype == mx.int32 else x.astype(mx.int32)
+    arr = _as_np(x)
+    if arr is None:
+        raise ValueError("Failed to convert to numpy for int32 conversion.")
+    return mx.array(arr.astype(np.int32))
+
+
+def _as_mx_bool(x: Any) -> Optional[mx.array]:
+    if x is None:
+        return None
+    if isinstance(x, mx.array):
+        return x if x.dtype == mx.bool_ else x.astype(mx.bool_)
+    arr = _as_np(x)
+    if arr is None:
+        raise ValueError("Failed to convert to numpy for bool conversion.")
+    return mx.array(arr.astype(np.bool_))
+
+
+# -----------------------------------------------------------------------------
+# Small utils
+# -----------------------------------------------------------------------------
+def l2_normalize(x: mx.array, eps: float = 1e-6) -> mx.array:
+    denom = mx.sqrt(mx.maximum((x * x).sum(axis=-1, keepdims=True), eps))
+    return x / denom
+
+
+def masked_scatter(
+    final_embedding: mx.array,
+    image_mask_expanded: mx.array,
+    scaled_image_features: mx.array,
+) -> mx.array:
+    """
+    Scatter image features into final_embedding where mask is True.
+    Compatible with MLX versions that do NOT support .at[].set().
+    """
+    final_shape = final_embedding.shape
+
+    img_flat = mx.flatten(scaled_image_features)
+    out_flat = mx.flatten(final_embedding)
+    mask_flat = mx.flatten(image_mask_expanded)
+
+    pos_np = np.where(np.array(mask_flat))[0].astype(np.uint32)
+    pos = mx.array(pos_np, dtype=mx.uint32)
+
+    #  MLX-compatible assignment (no .at[].set())
+    out_flat[pos] = img_flat
+
+    return mx.reshape(out_flat, final_shape)
+
+
+# -----------------------------------------------------------------------------
+# Backbone (Qwen3-VL style) for multimodal mixing + logits
+# -----------------------------------------------------------------------------
+class VLMBackbone(nn.Module):
+    """
+    Qwen3-VL backbone used by mlx-vlm:
+      - embeds tokens
+      - injects image features into <image>/<video> token positions
+      - runs LanguageModel to produce logits (for generation)
+    """
+
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        self.config = config
+        self.vision_tower = VisionModel(config.vision_config)
+        self.language_model = LanguageModel(config.text_config, config)
+
+    def get_input_embeddings(
+        self,
+        input_ids: Any,
+        pixel_values: Optional[Any] = None,
+        **kwargs,
+    ) -> dict:
+        """
+        Returns:
+          inputs_embeds: [B, T, H]
+          visual_pos_masks: [B, T] bool mask where visual tokens are
+          deepstack_visual_embeds: list[tensor] for deepstack (may be None)
+        """
+        input_ids_mx = _as_mx_int32(input_ids)
+
+        image_grid_thw = kwargs.get("image_grid_thw", None)
+        video_grid_thw = kwargs.get("video_grid_thw", None)
+        grid_thw = image_grid_thw if image_grid_thw is not None else video_grid_thw
+        if grid_thw is not None:
+            grid_thw = _as_mx_int32(grid_thw)
+
+        # text-only
+        if pixel_values is None:
+            return {
+                "inputs_embeds": self.language_model.model.embed_tokens(input_ids_mx),
+                "visual_pos_masks": None,
+                "deepstack_visual_embeds": None,
+            }
+
+        # ensure mx.array (torch tensor fix)
+        pixel_values_mx = _as_mx(pixel_values)
+
+        # cast to vision proj dtype
+        dtype = self.vision_tower.patch_embed.proj.weight.dtype
+        pixel_values_mx = pixel_values_mx.astype(dtype)
+
+        inputs_embeds = self.language_model.model.embed_tokens(input_ids_mx)
+        image_features, deepstack_image_embeds = self.vision_tower(pixel_values_mx, grid_thw)
+
+        inputs_embeds, image_mask = self.merge_input_ids_with_image_features(
+            image_features=image_features,
+            inputs_embeds=inputs_embeds,
+            input_ids=input_ids_mx,
+            image_token_index=self.config.image_token_index,
+            video_token_index=self.config.video_token_index,
+        )
+
+        visual_pos_masks = image_mask[..., 0].astype(mx.bool_)
+
+        return {
+            "inputs_embeds": inputs_embeds,
+            "visual_pos_masks": visual_pos_masks,
+            "deepstack_visual_embeds": deepstack_image_embeds,
+        }
+
+    @staticmethod
+    def merge_input_ids_with_image_features(
+        image_features: mx.array,
+        inputs_embeds: mx.array,
+        input_ids: mx.array,
+        image_token_index: int,
+        video_token_index: int,
+    ):
+        special_mask = (input_ids == image_token_index) | (input_ids == video_token_index)
+
+        special_mask = special_mask[..., None]
+        special_mask = mx.broadcast_to(special_mask, inputs_embeds.shape)
+
+        n_mask_elements = int(special_mask.sum().item()) if hasattr(special_mask.sum(), "item") else int(special_mask.sum())
+        if n_mask_elements != int(image_features.size):
+            raise ValueError(
+                f"Image features and image tokens do not match: "
+                f"mask_elems={n_mask_elements}, image_features.size={int(image_features.size)}"
+            )
+
+        out = masked_scatter(inputs_embeds, special_mask, image_features)
+        return out, special_mask
+
+    @property
+    def layers(self):
+        return self.language_model.model.layers
+
+    def __call__(
+        self,
+        input_ids: Any,
+        pixel_values: Optional[Any] = None,
+        mask: Optional[Any] = None,
+        cache=None,
+        **kwargs,
+    ):
+        pack = self.get_input_embeddings(input_ids=input_ids, pixel_values=pixel_values, **kwargs)
+        kwargs.update({"pixel_values": _as_mx(pixel_values), **pack})
+        logits = self.language_model(_as_mx_int32(input_ids), mask=_as_mx_int32(mask) if mask is not None else None, cache=cache, **kwargs)
+        return logits
+
+
+# -----------------------------------------------------------------------------
+# ColQwen3 wrapper (Tomoro) - token-level embeddings + MaxSim
+# -----------------------------------------------------------------------------
+class Model(nn.Module):
+    """
+    ColQwen3 MLX wrapper:
+      - keeps VLM backbone (logits path)
+      - adds embedding_proj_layer to output token embeddings [B,T,D]
+      - provides encode / encode_queries / encode_images and maxsim
+
+    IMPORTANT:
+      - deepstack injection is disabled by default here (use_deepstack=False),
+        because current deepstack wiring can mismatch shapes and crash on MLX/Metal.
+        Once language.py deepstack indexing is fixed, you can enable it.
+    """
+
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        self.config = config
+
+        # backbone behaves like qwen3_vl internally
+        backbone_cfg = replace(config, model_type="qwen3_vl")
+        self.vlm = VLMBackbone(backbone_cfg)
+
+        hidden = config.text_config.hidden_size
+        self.embedding_proj_layer = nn.Linear(hidden, config.embed_dim, bias=True)
+
+    # generation/logits path
+    def __call__(
+        self,
+        input_ids: Any,
+        pixel_values: Optional[Any] = None,
+        mask: Optional[Any] = None,
+        cache=None,
+        **kwargs,
+    ):
+        return self.vlm(_as_mx_int32(input_ids), pixel_values=pixel_values, mask=mask, cache=cache, **kwargs)
+
+    # -------------------------------------------------------------------------
+    # HF -> MLX weight key mapping
+    # -------------------------------------------------------------------------
+    def sanitize(self, weights: dict) -> dict:
+        out = {}
+        for k, v in weights.items():
+            # HF: vlm.model.language_model.* -> MLX: vlm.language_model.model.*
+            if k.startswith("vlm.model.language_model."):
+                nk = "vlm.language_model.model." + k[len("vlm.model.language_model.") :]
+                out[nk] = v
+                continue
+
+            # HF: vlm.lm_head.* -> MLX: vlm.language_model.lm_head.*
+            if k.startswith("vlm.lm_head."):
+                nk = "vlm.language_model.lm_head." + k[len("vlm.lm_head.") :]
+                out[nk] = v
+                continue
+
+            # HF: vlm.model.visual.* -> MLX: vlm.vision_tower.*
+            if k.startswith("vlm.model.visual."):
+                nk = "vlm.vision_tower." + k[len("vlm.model.visual.") :]
+                out[nk] = v
+                continue
+
+            # HF: vlm.model.vision_tower.* -> MLX: vlm.vision_tower.*
+            if k.startswith("vlm.model.vision_tower."):
+                nk = "vlm.vision_tower." + k[len("vlm.model.vision_tower.") :]
+                out[nk] = v
+                continue
+
+            # projection layer
+            if k.startswith("embedding_proj_layer."):
+                out[k] = v
+                continue
+
+            # fallback old names (just in case)
+            if k.startswith("model.language_model."):
+                nk = "vlm.language_model.model." + k[len("model.language_model.") :]
+                out[nk] = v
+                continue
+            if k.startswith("model.visual."):
+                nk = "vlm.vision_tower." + k[len("model.visual.") :]
+                out[nk] = v
+                continue
+
+            out[k] = v
+
+        return out
+
+    # -------------------------------------------------------------------------
+    # Embedding forward
+    # -------------------------------------------------------------------------
+    def encode(
+        self,
+        input_ids: Any,
+        pixel_values: Optional[Any] = None,
+        attention_mask: Optional[Any] = None,
+        only_visual: bool = False,
+        use_deepstack: bool = False,
+        **kwargs,
+    ) -> dict:
+        """
+        Returns token-level embeddings (ColBERT-style).
+
+        Output:
+          embeddings: [B,T,D] padded
+          embeddings_list: list length B, each [Ti,D] trimmed
+          attention_mask: mx.int32 if provided
+          visual_mask: [B,T] bool if multimodal
+        """
+        input_ids_mx = _as_mx_int32(input_ids)
+        attn_mx = _as_mx_int32(attention_mask) if attention_mask is not None else None
+        pixel_mx = _as_mx(pixel_values) if pixel_values is not None else None
+
+        # grids if present
+        if "image_grid_thw" in kwargs and kwargs["image_grid_thw"] is not None:
+            kwargs["image_grid_thw"] = _as_mx_int32(kwargs["image_grid_thw"])
+        if "video_grid_thw" in kwargs and kwargs["video_grid_thw"] is not None:
+            kwargs["video_grid_thw"] = _as_mx_int32(kwargs["video_grid_thw"])
+
+        pack = self.vlm.get_input_embeddings(input_ids=input_ids_mx, pixel_values=pixel_mx, **kwargs)
+
+        deepstack = pack.get("deepstack_visual_embeds", None) if use_deepstack else None
+        visual_mask = pack.get("visual_pos_masks", None)
+
+        h = self.vlm.language_model.forward_hidden(
+            input_ids_mx,
+            inputs_embeds=pack["inputs_embeds"],
+            mask=attn_mx,
+            visual_pos_masks=visual_mask,
+            deepstack_visual_embeds=deepstack,
+            pixel_values=pixel_mx,
+            **kwargs,
+        )
+
+        e = l2_normalize(self.embedding_proj_layer(h))
+        mx.eval(e)  # materialize to reduce lazy graph buildup / Metal crashes
+
+        # build per-sample lists
+        embs_list: List[mx.array] = []
+
+        if only_visual and (visual_mask is not None):
+            for b in range(int(e.shape[0])):
+                idx_np = np.where(np.array(visual_mask[b]))[0].astype(np.uint32)
+                idx_mx = mx.array(idx_np, dtype=mx.uint32)
+                embs_list.append(e[b][idx_mx])
+            return {
+                "embeddings": e,
+                "embeddings_list": embs_list,
+                "attention_mask": attn_mx,
+                "visual_mask": visual_mask,
+            }
+
+        if attn_mx is not None:
+            for b in range(int(e.shape[0])):
+                idx_np = np.where(np.array(attn_mx[b]))[0].astype(np.uint32)
+                idx_mx = mx.array(idx_np, dtype=mx.uint32)
+                embs_list.append(e[b][idx_mx])
+        else:
+            for b in range(int(e.shape[0])):
+                embs_list.append(e[b])
+
+        return {
+            "embeddings": e,
+            "embeddings_list": embs_list,
+            "attention_mask": attn_mx,
+            "visual_mask": visual_mask,
+        }
+
+    # -------------------------------------------------------------------------
+    # MaxSim scoring
+    # -------------------------------------------------------------------------
+    @staticmethod
+    def maxsim(q_emb: mx.array, d_emb: mx.array, chunk: int = 1024) -> mx.array:
+        """
+        ColBERT MaxSim (chunked + fp32 for stability):
+          score(q,d) = sum_i max_j <q_i, d_j>
+        """
+        q = q_emb.astype(mx.float32)
+        d = d_emb.astype(mx.float32)
+
+        Tq = int(q.shape[0])
+        Td = int(d.shape[0])
+
+        token_max = mx.full((Tq,), -1e9, dtype=mx.float32)
+
+        for j in range(0, Td, chunk):
+            dj = d[j : j + chunk]            # [c, D]
+            sim = q @ dj.T                   # [Tq, c]
+            token_max = mx.maximum(token_max, sim.max(axis=1))
+
+        s = token_max.sum()
+        mx.eval(s)
+        return s
+
+    # -------------------------------------------------------------------------
+    # Convenience helpers (processor integrated)
+    # -------------------------------------------------------------------------
+    def encode_queries(self, processor, texts: list[str], batch_size: int = 8, use_deepstack: bool = False):
+        """
+        Returns list of [T,D] per query (padding removed).
+        """
+        outs = []
+        for s in range(0, len(texts), batch_size):
+            batch = processor.process_texts(texts=texts[s : s + batch_size])
+
+            input_ids = _as_mx_int32(batch["input_ids"])
+            attn = _as_mx_int32(batch.get("attention_mask")) if "attention_mask" in batch else None
+
+            out = self.encode(
+                input_ids=input_ids,
+                attention_mask=attn,
+                only_visual=False,
+                use_deepstack=use_deepstack,
+            )
+            outs.extend(out["embeddings_list"])
+        return outs
+
+    def encode_images(self, processor, images, batch_size: int = 2, use_deepstack: bool = False):
+        """
+        Returns list of [Tv,D] per image (visual tokens only).
+        batch_size default=2 to reduce Metal memory pressure.
+        """
+        outs = []
+        for s in range(0, len(images), batch_size):
+            batch_imgs = images[s : s + batch_size]
+            feats = processor.process_images(images=batch_imgs)
+
+            input_ids = _as_mx_int32(feats["input_ids"])
+            pixel_values = _as_mx(feats.get("pixel_values"))
+            attn = _as_mx_int32(feats.get("attention_mask")) if "attention_mask" in feats else None
+
+            kwargs = {}
+            if "image_grid_thw" in feats and feats["image_grid_thw"] is not None:
+                kwargs["image_grid_thw"] = _as_mx_int32(feats["image_grid_thw"])
+            if "video_grid_thw" in feats and feats["video_grid_thw"] is not None:
+                kwargs["video_grid_thw"] = _as_mx_int32(feats["video_grid_thw"])
+
+            out = self.encode(
+                input_ids=input_ids,
+                pixel_values=pixel_values,
+                attention_mask=attn,
+                only_visual=True,
+                use_deepstack=use_deepstack,
+                **kwargs,
+            )
+            outs.extend(out["embeddings_list"])
+        return outs
\ No newline at end of file
diff --git a/mlx_vlm/models/colqwen3/config.py b/mlx_vlm/models/colqwen3/config.py
new file mode 100644
index 00000000..79af70b2
--- /dev/null
+++ b/mlx_vlm/models/colqwen3/config.py
@@ -0,0 +1,154 @@
+import inspect
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
+
+from ..base import BaseModelConfig
+
+def _filter_kwargs(cls, d: dict) -> dict:
+    allowed = set(inspect.signature(cls).parameters.keys())
+    return {k: v for k, v in d.items() if k in allowed}
+
+# -----------------------------
+# Vision Config
+# -----------------------------
+@dataclass
+class VisionConfig(BaseModelConfig):
+
+    model_type: str = "qwen3_vl"
+
+    depth: int = 32
+    hidden_size: int = 1280
+    intermediate_size: int = 3420
+    out_hidden_size: int = 1536
+    num_heads: int = 16
+
+    image_size: int = 384
+    patch_size: int = 14
+    vocab_size: int = 32000
+
+    mlp_ratio: float = 4.0
+    in_channels: int = 3
+    layer_norm_eps: float = 1e-6
+
+    spatial_patch_size: int = 14
+    spatial_merge_size: int = 2
+    tokens_per_second: int = 2
+    temporal_patch_size: int = 2
+
+    num_position_embeddings: int = 2304
+    window_size: int = 112
+
+    fullatt_block_indexes: list[int] = field(default_factory=lambda: [7, 15, 23, 31])
+    deepstack_visual_indexes: list[int] = field(default_factory=list)
+
+
+# -----------------------------
+# Text Config
+# -----------------------------
+@dataclass
+class TextConfig(BaseModelConfig):
+    model_type: str
+    num_hidden_layers: int
+    hidden_size: int
+    intermediate_size: int
+    num_attention_heads: int
+    rms_norm_eps: float
+    vocab_size: int
+
+    num_key_value_heads: Optional[int]
+    head_dim: int
+    rope_theta: float
+    max_position_embeddings: int
+
+    norm_topk_prob: bool = True
+    rope_scaling: Optional[Dict[str, Union[float, str, bool, List[int]]]] = field(
+        default_factory=lambda: {"type": "default", "mrope_section": [24, 20, 20]}
+    )
+
+    tie_word_embeddings: bool = False
+    attention_bias: bool = False
+    hidden_act: str = "silu"
+
+    def __post_init__(self):
+        if self.num_key_value_heads is None:
+            self.num_key_value_heads = self.num_attention_heads
+
+        if self.rope_scaling:
+
+            if "type" not in self.rope_scaling and "rope_type" in self.rope_scaling:
+                self.rope_scaling["type"] = self.rope_scaling.pop("rope_type")
+
+            required_keys = {"mrope_section", "type"}
+            if not all(k in self.rope_scaling for k in required_keys):
+                raise ValueError(f"rope_scaling must contain keys {required_keys}")
+
+            if self.rope_scaling["type"] not in ["mrope", "default"]:
+                raise ValueError("rope_scaling type must be 'mrope' or 'default'")
+
+
+# -----------------------------
+# ColQwen3 Model Config
+# -----------------------------
+@dataclass
+class ModelConfig(BaseModelConfig):
+
+    text_config: TextConfig
+    vision_config: VisionConfig
+    model_type: str
+
+    ignore_index: int = -100
+
+    image_token_id: int = 151655
+    video_token_id: int = 151656
+    image_token_index: Optional[int] = None
+    video_token_index: Optional[int] = None
+
+    vision_start_token_id: int = 151652
+    vision_end_token_id: int = 151653
+    vision_token_id: int = 151654
+
+    vision_feature_select_strategy: str = "default"
+    vision_feature_layer: int = -2
+
+    vocab_size: int = 32000
+    eos_token_id: Optional[List[int]] = None
+
+    embed_dim: int = 320
+
+    def __post_init__(self):
+        if self.image_token_index is None:
+            self.image_token_index = self.image_token_id
+        if self.video_token_index is None:
+            self.video_token_index = self.video_token_id
+
+    @classmethod
+    def from_dict(cls, params: dict):
+        """
+        HF config.json -> dataclass
+
+        """
+
+        tc = params.get("text_config")
+        vc = params.get("vision_config")
+
+        if isinstance(tc, dict):
+            tc = TextConfig(**_filter_kwargs(TextConfig, tc))
+        elif not isinstance(tc, TextConfig):
+            raise ValueError(f"text_config must be dict or TextConfig, got {type(tc)}")
+
+        if isinstance(vc, dict):
+            vc = VisionConfig(**_filter_kwargs(VisionConfig, vc))
+        elif not isinstance(vc, VisionConfig):
+            raise ValueError(f"vision_config must be dict or VisionConfig, got {type(vc)}")
+
+        allowed = set(inspect.signature(cls).parameters.keys())
+        kwargs = {k: v for k, v in params.items() if k in allowed}
+
+        # override with parsed configs
+        kwargs["text_config"] = tc
+        kwargs["vision_config"] = vc
+
+        if "embed_dim" in params:
+            kwargs["embed_dim"] = int(params["embed_dim"])
+
+        return cls(**kwargs)
\ No newline at end of file
diff --git a/mlx_vlm/models/colqwen3/language.py b/mlx_vlm/models/colqwen3/language.py
new file mode 100644
index 00000000..7e091411
--- /dev/null
+++ b/mlx_vlm/models/colqwen3/language.py
@@ -0,0 +1,549 @@
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+import numpy as np
+
+from ..base import (
+    LanguageModelOutput,
+    create_attention_mask,
+    scaled_dot_product_attention,
+)
+from ..cache import KVCache
+from .config import ModelConfig, TextConfig
+
+
+# -----------------------------------------------------------------------------
+# Rotary embeddings (Qwen3-VL MRoPE)
+# -----------------------------------------------------------------------------
+class Qwen3VLRotaryEmbedding:
+    def __init__(
+        self,
+        dim: int,
+        max_position_embeddings: int = 2048,
+        base: float = 10000.0,
+        rope_scaling: Optional[dict] = None,
+    ):
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+
+        inv_freq = 1.0 / (
+            self.base ** (mx.arange(0, self.dim, 2).astype(mx.float32) / self.dim)
+        )
+        self.inv_freq = inv_freq
+
+        rope_scaling = rope_scaling or {}
+        self.mrope_section = rope_scaling.get("mrope_section", [24, 20, 20])
+
+    def apply_interleaved_mrope(self, freqs: mx.array, mrope_section) -> mx.array:
+        """
+        freqs: (3, bs, seq_len, head_dim//2)
+        Returns: (bs, seq_len, head_dim//2) with interleaved layout.
+        """
+        freqs_t = freqs[0]  # overwrite T lane
+        for dim, offset in enumerate((1, 2), start=1):  # H, W lanes
+            length = mrope_section[dim] * 3
+            idx = slice(offset, length, 3)
+            freqs_t[..., idx] = freqs[dim, ..., idx]
+        return freqs_t
+
+    def __call__(self, x: mx.array, position_ids: mx.array) -> Tuple[mx.array, mx.array]:
+        # position_ids can be (B, T) or (3, B, T)
+        if position_ids.ndim == 2:
+            position_ids = mx.broadcast_to(
+                position_ids[None, ...],
+                (3, position_ids.shape[0], position_ids.shape[1]),
+            )
+
+        inv_freq_expanded = mx.broadcast_to(
+            self.inv_freq[None, None, :, None].astype(mx.float32),
+            (3, position_ids.shape[1], self.inv_freq.shape[0], 1),
+        )
+        pos_expanded = position_ids[:, :, None, :].astype(mx.float32)  # (3, B, 1, T)
+
+        freqs = inv_freq_expanded @ pos_expanded  # (3, B, dim/2, T)
+        freqs = mx.swapaxes(freqs, 2, 3)          # (3, B, T, dim/2)
+        freqs = self.apply_interleaved_mrope(freqs, self.mrope_section)
+
+        emb = mx.concatenate([freqs, freqs], axis=-1)
+        cos = mx.cos(emb).astype(x.dtype)
+        sin = mx.sin(emb).astype(x.dtype)
+        return cos, sin
+
+
+def rotate_half(x: mx.array) -> mx.array:
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return mx.concatenate([-x2, x1], axis=-1)
+
+
+def apply_multimodal_rotary_pos_emb(
+    q: mx.array, k: mx.array, cos: mx.array, sin: mx.array, unsqueeze_dim: int = 1
+) -> Tuple[mx.array, mx.array]:
+    cos = mx.expand_dims(cos, axis=unsqueeze_dim)
+    sin = mx.expand_dims(sin, axis=unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+# -----------------------------------------------------------------------------
+# Transformer blocks
+# -----------------------------------------------------------------------------
+class Attention(nn.Module):
+    def __init__(self, args: TextConfig):
+        super().__init__()
+
+        dim = args.hidden_size
+        self.n_heads = args.num_attention_heads
+        self.n_kv_heads = args.num_key_value_heads or args.num_attention_heads
+
+        self.head_dim = getattr(args, "head_dim", args.hidden_size // args.num_attention_heads)
+        self.scale = self.head_dim**-0.5
+
+        self.q_proj = nn.Linear(dim, self.n_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(dim, self.n_kv_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(dim, self.n_kv_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.n_heads * self.head_dim, dim, bias=False)
+
+        self.q_norm = nn.RMSNorm(dims=self.head_dim, eps=args.rms_norm_eps)
+        self.k_norm = nn.RMSNorm(dims=self.head_dim, eps=args.rms_norm_eps)
+
+        self.rotary_emb = Qwen3VLRotaryEmbedding(
+            self.head_dim,
+            max_position_embeddings=args.max_position_embeddings,
+            base=args.rope_theta,
+            rope_scaling=args.rope_scaling,
+        )
+
+    def __call__(
+        self,
+        x: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[KVCache] = None,
+        position_ids: Optional[mx.array] = None,
+    ) -> mx.array:
+        B, L, _ = x.shape
+
+        q = self.q_proj(x)
+        k = self.k_proj(x)
+        v = self.v_proj(x)
+
+        q = self.q_norm(q.reshape(B, L, self.n_heads, self.head_dim)).transpose(0, 2, 1, 3)
+        k = self.k_norm(k.reshape(B, L, self.n_kv_heads, self.head_dim)).transpose(0, 2, 1, 3)
+        v = v.reshape(B, L, self.n_kv_heads, self.head_dim).transpose(0, 2, 1, 3)
+
+        cache_offset = 0
+        if cache is not None:
+            # KVCache.offset exists
+            off = cache.offset
+            if isinstance(off, int):
+                cache_offset = off
+            elif isinstance(off, mx.array):
+                cache_offset = (off if off.ndim == 0 else off[0]).item()
+            else:
+                cache_offset = int(off)
+
+        kv_seq_len = k.shape[-2] + (cache_offset + 1 if cache is not None else 0)
+
+        if position_ids is None:
+            # default position ids
+            start = cache_offset
+            pos = mx.arange(start, start + L)
+            pos = mx.expand_dims(pos, axis=0)      # (1, L)
+            pos = mx.tile(pos, (3, 1, 1))          # (3, 1, L)
+            position_ids = pos
+
+        cos, sin = self.rotary_emb(v, position_ids)
+
+        if mask is not None and isinstance(mask, mx.array):
+            mask = mask[..., :kv_seq_len]
+
+        q, k = apply_multimodal_rotary_pos_emb(q, k, cos, sin)
+
+        if cache is not None:
+            k, v = cache.update_and_fetch(k, v)
+
+        out = scaled_dot_product_attention(q, k, v, cache, scale=self.scale, mask=mask)
+        out = out.transpose(0, 2, 1, 3).reshape(B, L, -1)
+        return self.o_proj(out)
+
+
+class MLP(nn.Module):
+    def __init__(self, dim: int, hidden_dim: int):
+        super().__init__()
+        self.gate_proj = nn.Linear(dim, hidden_dim, bias=False)
+        self.up_proj = nn.Linear(dim, hidden_dim, bias=False)
+        self.down_proj = nn.Linear(hidden_dim, dim, bias=False)
+
+    def __call__(self, x: mx.array) -> mx.array:
+        return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class Qwen3VLDecoderLayer(nn.Module):
+    def __init__(self, args: TextConfig, layer_idx: int):
+        super().__init__()
+        self.self_attn = Attention(args)
+        self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+        self.post_attention_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+        self.mlp = MLP(args.hidden_size, args.intermediate_size)
+
+    def __call__(
+        self,
+        x: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[KVCache] = None,
+        position_ids: Optional[mx.array] = None,
+    ) -> mx.array:
+        r = self.self_attn(self.input_layernorm(x), mask=mask, cache=cache, position_ids=position_ids)
+        h = x + r
+        r = self.mlp(self.post_attention_layernorm(h))
+        return h + r
+
+
+class Qwen3VLModel(nn.Module):
+    def __init__(self, args: TextConfig):
+        super().__init__()
+        self.args = args
+        self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
+        self.layers = [Qwen3VLDecoderLayer(args, i) for i in range(args.num_hidden_layers)]
+        self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+
+    def _deepstack_process(
+            self,
+            hidden_states: mx.array,
+            visual_pos_masks: mx.array,
+            visual_embeds: mx.array,
+    ):
+        """
+        hidden_states: [B, T, H]
+        visual_pos_masks: [B, T] bool-like
+        visual_embeds: [Nvis, H]  (per sample)
+        """
+        batch_size = hidden_states.shape[0]
+        updated_batches = []
+
+        for b in range(batch_size):
+            mask_b = visual_pos_masks[b]
+            hb = mx.array(hidden_states[b])  # avoid in-place issues
+
+            idx_np = np.where(np.array(mask_b))[0].astype(np.uint32)
+            if idx_np.size == 0:
+                updated_batches.append(hb)
+                continue
+
+            idx_mx = mx.array(idx_np, dtype=mx.uint32)
+
+            ve = visual_embeds
+            n_idx = int(idx_np.size)
+            n_vis = int(ve.shape[0])
+
+            # ---- align lengths (fixes your 2450 vs 1225 case) ----
+            if n_idx != n_vis:
+                if n_vis > 0 and (n_idx % n_vis == 0):
+                    reps = n_idx // n_vis
+                    ve = mx.tile(ve, (reps, 1))  # [n_idx, H]
+                else:
+                    n = min(n_idx, n_vis)
+                    idx_mx = idx_mx[:n]
+                    ve = ve[:n]
+
+            hb = hb.at[idx_mx].add(ve)
+            updated_batches.append(hb)
+
+        return mx.stack(updated_batches, axis=0)
+
+    def __call__(
+        self,
+        inputs: mx.array,
+        inputs_embeds: Optional[mx.array] = None,
+        mask: Optional[mx.array] = None,
+        cache=None,
+        position_ids: Optional[mx.array] = None,
+        visual_pos_masks: Optional[mx.array] = None,
+        deepstack_visual_embeds: Optional[list[mx.array]] = None,
+    ) -> mx.array:
+        h = self.embed_tokens(inputs) if inputs_embeds is None else inputs_embeds
+
+        if cache is None:
+            cache = [None] * len(self.layers)
+
+        if mask is None:
+            mask = create_attention_mask(h, cache)
+
+        for layer_idx, (layer, c) in enumerate(zip(self.layers, cache)):
+            h = layer(h, mask=mask, cache=c, position_ids=position_ids)
+
+            if deepstack_visual_embeds is not None and visual_pos_masks is not None:
+                if layer_idx < len(deepstack_visual_embeds):
+                    h = self._deepstack_process(h, visual_pos_masks, deepstack_visual_embeds[layer_idx])
+
+        return self.norm(h)
+
+
+# -----------------------------------------------------------------------------
+# LanguageModel wrapper (adds RoPE index logic + lm_head)
+# -----------------------------------------------------------------------------
+class LanguageModel(nn.Module):
+    def __init__(self, args: TextConfig, config: Optional[ModelConfig] = None):
+        super().__init__()
+        self.args = args
+        self.config = config
+        self.model = Qwen3VLModel(args)
+        self._rope_deltas = None
+
+        if not args.tie_word_embeddings:
+            self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)
+
+    # --- RoPE index helper (kept from your implementation, just cleaned a bit) ---
+    def get_rope_index(
+        self,
+        input_ids: mx.array,
+        image_grid_thw: Optional[mx.array] = None,
+        video_grid_thw: Optional[mx.array] = None,
+        attention_mask: Optional[mx.array] = None,
+    ):
+        batch_size, seq_length = input_ids.shape
+
+        position_ids = mx.arange(seq_length, dtype=mx.int32)
+        position_ids = mx.broadcast_to(position_ids[None, :], (batch_size, seq_length))
+
+        spatial_merge_size = self.config.vision_config.spatial_merge_size
+        image_token_id = self.config.image_token_id
+        video_token_id = self.config.video_token_id
+        vision_start_token_id = self.config.vision_start_token_id
+
+        if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None):
+            total_input_ids = input_ids
+
+            if attention_mask is None:
+                attention_mask = mx.ones_like(input_ids)
+
+            position_ids = mx.ones((3, input_ids.shape[0], input_ids.shape[1]), dtype=input_ids.dtype)
+
+            image_index, video_index = 0, 0
+            mrope_position_deltas = []
+
+            for i, ids_i in enumerate(total_input_ids):
+                ids_i = mx.where(attention_mask[i] == 1, ids_i, mx.zeros_like(ids_i))
+
+                # find vision start indices (works as in original code)
+                vision_start_indices = mx.sum(
+                    mx.where(
+                        ids_i == vision_start_token_id,
+                        mx.arange(ids_i.shape[0]),
+                        mx.zeros_like(ids_i),
+                    )
+                )
+                vision_tokens = ids_i[vision_start_indices + 1]
+                image_nums = (vision_tokens == image_token_id).sum().item()
+                video_nums = (vision_tokens == video_token_id).sum().item()
+
+                input_tokens = ids_i.tolist()
+                llm_pos_ids_list = []
+                st = 0
+                remain_images, remain_videos = image_nums, video_nums
+
+                for _ in range(image_nums + video_nums):
+                    if image_token_id in input_tokens and remain_images > 0:
+                        ed_image = input_tokens.index(image_token_id, st)
+                    else:
+                        ed_image = len(input_tokens) + 1
+
+                    if video_token_id in input_tokens and remain_videos > 0:
+                        ed_video = input_tokens.index(video_token_id, st)
+                    else:
+                        ed_video = len(input_tokens) + 1
+
+                    if ed_image < ed_video:
+                        t, h, w = image_grid_thw[image_index]
+                        image_index += 1
+                        remain_images -= 1
+                        ed = ed_image
+                    else:
+                        t, h, w = video_grid_thw[video_index]
+                        video_index += 1
+                        remain_videos -= 1
+                        ed = ed_video
+
+                    llm_grid_t = int(t.item())
+                    llm_grid_h = int(h.item()) // spatial_merge_size
+                    llm_grid_w = int(w.item()) // spatial_merge_size
+
+                    text_len = ed - st
+                    st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+
+                    index = mx.arange(text_len).reshape(1, text_len)
+                    index = mx.broadcast_to(index, (3, text_len))
+                    index = index + st_idx
+                    llm_pos_ids_list.append(index)
+
+                    t_index = mx.arange(llm_grid_t).reshape(llm_grid_t, 1)
+                    t_index = mx.broadcast_to(t_index, (llm_grid_t, llm_grid_h * llm_grid_w)).flatten()
+
+                    h_index = mx.arange(llm_grid_h).reshape(1, llm_grid_h, 1)
+                    h_index = mx.broadcast_to(h_index, (llm_grid_t, llm_grid_h, llm_grid_w)).flatten()
+
+                    w_index = mx.arange(llm_grid_w).reshape(1, 1, llm_grid_w)
+                    w_index = mx.broadcast_to(w_index, (llm_grid_t, llm_grid_h, llm_grid_w)).flatten()
+
+                    llm_pos_ids_list.append(
+                        mx.stack([t_index, h_index, w_index]) + text_len + st_idx
+                    )
+
+                    st = ed + llm_grid_t * llm_grid_h * llm_grid_w
+
+                if st < len(input_tokens):
+                    st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+                    text_len = len(input_tokens) - st
+
+                    t_index = mx.arange(text_len).reshape(1, text_len)
+                    t_index = mx.broadcast_to(t_index, (3, text_len))
+                    llm_pos_ids_list.append(t_index + st_idx)
+
+                llm_positions = mx.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
+
+                mask_i = mx.array(attention_mask[i] == 1)
+                expanded_mask = mx.expand_dims(mask_i, axis=0)
+                expanded_mask = mx.broadcast_to(expanded_mask, (3, 1, mask_i.shape[0]))
+
+                expanded_positions = mx.expand_dims(llm_positions, axis=1)
+                new_positions = mx.where(expanded_mask, expanded_positions, position_ids[:, i : i + 1, :])
+
+                position_ids = mx.concatenate(
+                    [position_ids[:, :i, :], new_positions, position_ids[:, i + 1 :, :]],
+                    axis=1,
+                )
+
+                mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i]))
+
+            mrope_position_deltas = mx.array(mrope_position_deltas)[0]
+            return position_ids, mrope_position_deltas
+
+        # text-only fallback
+        if attention_mask is not None:
+            position_ids = mx.cumsum(attention_mask.astype(mx.int64), axis=-1) - 1
+            position_ids = mx.where(attention_mask == 0, mx.ones_like(position_ids), position_ids)
+            position_ids = mx.expand_dims(position_ids[0], axis=0)
+            position_ids = mx.tile(position_ids, (3, 1, 1))
+
+            max_position_ids = position_ids.max(0, keepdims=False)[0].max(-1, keepdims=True)[0]
+            mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]
+        else:
+            position_ids = mx.arange(input_ids.shape[1]).reshape(1, -1)
+            position_ids = mx.broadcast_to(position_ids, (3, input_ids.shape[0], input_ids.shape[1]))
+            mrope_position_deltas = mx.zeros([input_ids.shape[0], 1], dtype=input_ids.dtype)
+
+        return position_ids, mrope_position_deltas
+
+    # --- key: hidden-only forward for embedding head usage ---
+    def forward_hidden(
+        self,
+        inputs: mx.array,
+        inputs_embeds: Optional[mx.array] = None,
+        mask: Optional[mx.array] = None,
+        cache=None,
+        visual_pos_masks: Optional[mx.array] = None,
+        deepstack_visual_embeds: Optional[list[mx.array]] = None,
+        **kwargs,
+    ) -> mx.array:
+        # prefill slicing
+        n_to_process = kwargs.get("n_to_process", None)
+        if n_to_process is not None and visual_pos_masks is not None:
+            visual_pos_masks = visual_pos_masks[:, n_to_process:]
+
+        position_ids = kwargs.pop("position_ids", None)
+        pixel_values = kwargs.pop("pixel_values", None)
+        image_grid_thw = kwargs.pop("image_grid_thw", None)
+        video_grid_thw = kwargs.pop("video_grid_thw", None)
+
+        if pixel_values is not None:
+            self._rope_deltas = None
+
+        cache_offset = 0
+        if cache and cache[0] is not None:
+            off = cache[0].offset
+            if isinstance(off, int):
+                cache_offset = off
+            elif isinstance(off, mx.array):
+                cache_offset = (off if off.ndim == 0 else off[0]).item()
+            else:
+                cache_offset = int(off)
+
+        if position_ids is None and (mask is None or mask.ndim == 2):
+            if (
+                (cache is not None and cache[0] is not None and cache_offset == 0)
+                or self._rope_deltas is None
+                or cache is None
+            ):
+                position_ids, rope_deltas = self.get_rope_index(inputs, image_grid_thw, video_grid_thw, mask)
+                self._rope_deltas = rope_deltas
+            else:
+                batch_size, seq_length = inputs.shape
+                delta = mx.array(cache_offset + self._rope_deltas if cache is not None else 0)
+                pos = mx.arange(seq_length).reshape(1, -1)
+                pos = mx.broadcast_to(pos, (batch_size, seq_length))
+
+                if delta.ndim == 0:
+                    delta = mx.expand_dims(delta, axis=0)
+                if delta.shape[0] < batch_size:
+                    delta = mx.tile(delta, (batch_size, 1))
+                else:
+                    delta = delta[:batch_size]
+
+                pos = mx.add(pos, delta)[None, ...]
+                position_ids = mx.broadcast_to(pos, (3, batch_size, seq_length))
+
+        h = self.model(
+            inputs,
+            cache=cache,
+            inputs_embeds=inputs_embeds,
+            position_ids=position_ids,
+            visual_pos_masks=visual_pos_masks,
+            deepstack_visual_embeds=deepstack_visual_embeds,
+        )
+        return h
+
+    def __call__(
+        self,
+        inputs: mx.array,
+        inputs_embeds: Optional[mx.array] = None,
+        mask: Optional[mx.array] = None,
+        cache=None,
+        visual_pos_masks: Optional[mx.array] = None,
+        deepstack_visual_embeds: Optional[list[mx.array]] = None,
+        **kwargs,
+    ) -> LanguageModelOutput:
+        h = self.forward_hidden(
+            inputs,
+            inputs_embeds=inputs_embeds,
+            mask=mask,
+            cache=cache,
+            visual_pos_masks=visual_pos_masks,
+            deepstack_visual_embeds=deepstack_visual_embeds,
+            **kwargs,
+        )
+
+        if self.args.tie_word_embeddings:
+            logits = self.model.embed_tokens.as_linear(h)
+        else:
+            logits = self.lm_head(h)
+
+        return LanguageModelOutput(logits=logits)
+
+    @property
+    def layers(self):
+        return self.model.layers
+
+    @property
+    def head_dim(self):
+        return self.args.hidden_size // self.args.num_attention_heads
+
+    @property
+    def n_kv_heads(self):
+        return self.args.num_key_value_heads or self.args.num_attention_heads
+
+
diff --git a/mlx_vlm/models/colqwen3/vision.py b/mlx_vlm/models/colqwen3/vision.py
new file mode 100644
index 00000000..2d13c02d
--- /dev/null
+++ b/mlx_vlm/models/colqwen3/vision.py
@@ -0,0 +1,441 @@
+from itertools import accumulate
+
+import mlx.core as mx
+import mlx.nn as nn
+
+from .config import VisionConfig
+
+
+def check_array_shape(arr):
+    shape = arr.shape
+
+    # Check if the shape has 4 or 5 dimensions
+    if len(shape) not in [4, 5]:
+        return False
+
+    B, out_channels, kH, KW, t = shape
+
+    if t == 3:
+        return True
+
+    # Check if out_channels is the largest, and kH and KW are the same
+    if (out_channels >= kH) and (out_channels >= KW) and (kH == KW):
+        return True
+    else:
+        return False
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return mx.concatenate([-x2, x1], axis=-1)
+
+
+def apply_rotary_pos_emb_vision(tensor, freqs) -> mx.array:
+    orig_dtype = tensor.dtype
+
+    cos = mx.cos(freqs)
+    sin = mx.sin(freqs)
+
+    cos = mx.expand_dims(cos, axis=1)
+    cos = mx.tile(cos, (1, 1, 2))
+    cos = mx.expand_dims(cos, axis=0)
+
+    sin = mx.expand_dims(sin, axis=1)
+    sin = mx.tile(sin, (1, 1, 2))
+    sin = mx.expand_dims(sin, axis=0)
+
+    output = (tensor * cos) + (rotate_half(tensor) * sin)
+    return output.astype(orig_dtype)
+
+
+class VisionRotaryEmbedding(nn.Module):
+    def __init__(self, dim: int, theta: float = 10000.0) -> None:
+        super().__init__()
+        self.dim = dim
+        self.theta = theta
+
+    def __call__(self, seqlen: int) -> mx.array:
+        inv_freq = 1.0 / (
+            self.theta ** (mx.arange(0, self.dim, 2, dtype=mx.float32) / self.dim)
+        )
+        seq = mx.arange(seqlen, dtype=inv_freq.dtype)
+        freqs = mx.outer(seq, inv_freq)
+        return freqs
+
+
+class PatchEmbed(nn.Module):
+    def __init__(
+        self,
+        patch_size: int = 14,
+        temporal_patch_size: int = 2,
+        in_channels: int = 3,
+        hidden_size: int = 1152,
+    ) -> None:
+        super().__init__()
+        self.patch_size = patch_size
+        self.temporal_patch_size = temporal_patch_size
+        self.in_channels = in_channels
+        self.hidden_size = hidden_size
+
+        kernel_size = [temporal_patch_size, patch_size, patch_size]
+        self.proj = nn.Conv3d(
+            in_channels,
+            hidden_size,
+            kernel_size=kernel_size,
+            stride=kernel_size,
+            bias=True,
+        )
+
+    def __call__(self, hidden_states: mx.array) -> mx.array:
+        hidden_states = hidden_states.reshape(
+            -1,
+            self.in_channels,
+            self.temporal_patch_size,
+            self.patch_size,
+            self.patch_size,
+        ).moveaxis(1, 4)
+
+        hidden_states = self.proj(hidden_states)
+        hidden_states = hidden_states.reshape(-1, self.hidden_size)
+        return hidden_states
+
+
+class PatchMerger(nn.Module):
+    def __init__(self, config: VisionConfig, use_postshuffle_norm=False) -> None:
+        super().__init__()
+        self.hidden_size = config.hidden_size * (config.spatial_merge_size**2)
+        self.use_postshuffle_norm = use_postshuffle_norm
+        self.norm = nn.LayerNorm(
+            self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6
+        )
+        self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)
+        self.act_fn = nn.GELU()
+        self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size)
+
+    def __call__(self, x: mx.array) -> mx.array:
+        x = self.norm(
+            x.reshape(-1, self.hidden_size) if self.use_postshuffle_norm else x
+        ).reshape(-1, self.hidden_size)
+        x = self.linear_fc2(self.act_fn(self.linear_fc1(x)))
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(self, dim: int, num_heads: int = 16) -> None:
+        super().__init__()
+        self.num_heads = num_heads
+        self.head_dim = head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+        self.qkv = nn.Linear(dim, dim * 3, bias=True)
+        self.proj = nn.Linear(dim, dim)
+
+    def __call__(
+        self, x: mx.array, cu_seqlens: mx.array, rotary_pos_emb: mx.array = None
+    ) -> mx.array:
+        seq_length = x.shape[0]
+        qkv = (
+            self.qkv(x).reshape(seq_length, 3, self.num_heads, -1).transpose(1, 0, 2, 3)
+        )
+        q, k, v = mx.split(qkv, 3)
+
+        q = apply_rotary_pos_emb_vision(mx.expand_dims(q, 0), rotary_pos_emb)[0]
+        k = apply_rotary_pos_emb_vision(mx.expand_dims(k, 0), rotary_pos_emb)[0]
+
+        q = q.transpose(0, 2, 1, 3)
+        k = k.transpose(0, 2, 1, 3)
+        v = v.transpose(0, 2, 1, 3)
+
+        splits = [
+            mx.split(tensor, cu_seqlens[1:-1].tolist(), axis=2) for tensor in (q, k, v)
+        ]
+
+        attn_outputs = []
+        for q, k, v in zip(*splits):
+            output = mx.fast.scaled_dot_product_attention(
+                q, k, v, scale=self.scale, mask=None
+            )
+            attn_outputs.append(output)
+
+        output = mx.concatenate(attn_outputs, axis=2)
+        output = output.transpose(0, 2, 1, 3).reshape(seq_length, -1)
+        return self.proj(output)
+
+
+class MLP(nn.Module):
+    def __init__(self, dim, hidden_dim):
+        super().__init__()
+        self.linear_fc1 = nn.Linear(dim, hidden_dim, bias=True)
+        self.linear_fc2 = nn.Linear(hidden_dim, dim, bias=True)
+        self.act_fn = nn.GELU(approx="tanh")
+
+    def __call__(self, x: mx.array) -> mx.array:
+        return self.linear_fc2(self.act_fn(self.linear_fc1(x)))
+
+
+class Qwen3VLMoEVisionBlock(nn.Module):
+    def __init__(self, config: VisionConfig) -> None:
+        super().__init__()
+        self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)
+        self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)
+
+        self.attn = Attention(dim=config.hidden_size, num_heads=config.num_heads)
+        self.mlp = MLP(dim=config.hidden_size, hidden_dim=config.intermediate_size)
+
+    def __call__(self, hidden_states, cu_seqlens, rotary_pos_emb) -> mx.array:
+        hidden_states = hidden_states + self.attn(
+            self.norm1(hidden_states),
+            cu_seqlens=cu_seqlens,
+            rotary_pos_emb=rotary_pos_emb,
+        )
+        hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))
+        return hidden_states
+
+
+class VisionModel(nn.Module):
+    def __init__(self, config: VisionConfig) -> None:
+        super().__init__()
+        self.config = config
+        self.model_type = config.model_type
+
+        if self.model_type != "qwen3_vl":
+            raise ValueError(f"Unsupported model type: {self.model_type}")
+
+        self.spatial_merge_size = config.spatial_merge_size
+
+        self.patch_embed = PatchEmbed(
+            patch_size=config.patch_size,
+            temporal_patch_size=config.temporal_patch_size,
+            in_channels=config.in_channels,
+            hidden_size=config.hidden_size,
+        )
+
+        head_dim = config.hidden_size // config.num_heads
+        self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2)
+
+        self.pos_embed = nn.Embedding(
+            config.num_position_embeddings, config.hidden_size
+        )
+        self.num_grid_per_side = int(config.num_position_embeddings**0.5)
+
+        self.blocks = [Qwen3VLMoEVisionBlock(config) for _ in range(config.depth)]
+        self.merger = PatchMerger(config=config, use_postshuffle_norm=False)
+
+        self.deepstack_visual_indexes = config.deepstack_visual_indexes
+        self.deepstack_merger_list = [
+            PatchMerger(
+                config=config,
+                use_postshuffle_norm=True,
+            )
+            for _ in range(len(config.deepstack_visual_indexes))
+        ]
+
+    def rot_pos_emb(self, grid_thw: mx.array) -> mx.array:
+        merge_size = self.spatial_merge_size
+
+        # Get max grid size for frequency table
+        max_hw = int(mx.max(grid_thw[:, 1:]).item())
+        freq_table = self.rotary_pos_emb(max_hw)  # Shape: (max_hw, dim // 2)
+
+        pos_ids = []
+
+        for num_frames, height, width in grid_thw.tolist():
+            num_frames, height, width = int(num_frames), int(height), int(width)
+            merged_h, merged_w = height // merge_size, width // merge_size
+
+            # Create block indices
+            block_rows = mx.arange(merged_h)
+            block_cols = mx.arange(merged_w)
+
+            # Create intra-block indices
+            intra_row = mx.arange(merge_size)
+            intra_col = mx.arange(merge_size)
+
+            # Compute full-resolution positions
+            row_idx = (
+                block_rows[:, None, None, None] * merge_size
+                + intra_row[None, None, :, None]
+            )
+            col_idx = (
+                block_cols[None, :, None, None] * merge_size
+                + intra_col[None, None, None, :]
+            )
+
+            # Broadcast and flatten
+            row_idx = mx.broadcast_to(
+                row_idx, (merged_h, merged_w, merge_size, merge_size)
+            ).reshape(-1)
+            col_idx = mx.broadcast_to(
+                col_idx, (merged_h, merged_w, merge_size, merge_size)
+            ).reshape(-1)
+
+            # Stack into coordinate pairs
+            coords = mx.stack([row_idx, col_idx], axis=-1)
+
+            # Repeat for temporal dimension
+            if num_frames > 1:
+                coords = mx.tile(coords, (num_frames, 1))
+
+            pos_ids.append(coords)
+
+        # Concatenate all position IDs - shape: (total_tokens, 2)
+        pos_ids = mx.concatenate(pos_ids, axis=0)
+
+        # Lookup embeddings: freq_table[h_pos] and freq_table[w_pos]
+        # pos_ids[:, 0] = height positions, pos_ids[:, 1] = width positions
+        h_embeddings = freq_table[pos_ids[:, 0]]  # (total_tokens, dim // 2)
+        w_embeddings = freq_table[pos_ids[:, 1]]  # (total_tokens, dim // 2)
+
+        # Concatenate height and width embeddings
+        embeddings = mx.concatenate([h_embeddings, w_embeddings], axis=-1)
+
+        return embeddings
+
+    def fast_pos_embed_interpolate(self, grid_thw):
+        grid_thw_list = grid_thw.tolist()
+        idx_list = [[] for _ in range(4)]
+        weight_list = [[] for _ in range(4)]
+
+        for t, h, w in grid_thw_list:
+            h = int(h)
+            w = int(w)
+            t = int(t)
+
+            h_idxs = mx.linspace(0, self.num_grid_per_side - 1, h)
+            w_idxs = mx.linspace(0, self.num_grid_per_side - 1, w)
+
+            h_idxs_floor = h_idxs.astype(mx.int32)
+            w_idxs_floor = w_idxs.astype(mx.int32)
+            h_idxs_ceil = mx.minimum(h_idxs_floor + 1, self.num_grid_per_side - 1)
+            w_idxs_ceil = mx.minimum(w_idxs_floor + 1, self.num_grid_per_side - 1)
+
+            dh = h_idxs - h_idxs_floor.astype(mx.float32)
+            dw = w_idxs - w_idxs_floor.astype(mx.float32)
+
+            base_h = h_idxs_floor * self.num_grid_per_side
+            base_h_ceil = h_idxs_ceil * self.num_grid_per_side
+
+            indices = [
+                (base_h[:, None] + w_idxs_floor[None, :]).flatten(),
+                (base_h[:, None] + w_idxs_ceil[None, :]).flatten(),
+                (base_h_ceil[:, None] + w_idxs_floor[None, :]).flatten(),
+                (base_h_ceil[:, None] + w_idxs_ceil[None, :]).flatten(),
+            ]
+
+            weights = [
+                ((1 - dh)[:, None] * (1 - dw)[None, :]).flatten(),
+                ((1 - dh)[:, None] * dw[None, :]).flatten(),
+                (dh[:, None] * (1 - dw)[None, :]).flatten(),
+                (dh[:, None] * dw[None, :]).flatten(),
+            ]
+
+            for i in range(4):
+                idx_list[i].extend(indices[i].tolist())
+                weight_list[i].extend(weights[i].tolist())
+
+        idx_tensor = mx.array(idx_list, dtype=mx.int32)
+        weight_tensor = mx.array(weight_list, dtype=self.pos_embed.weight.dtype)
+
+        pos_embeds = self.pos_embed(idx_tensor) * weight_tensor[:, :, None]
+        patch_pos_embeds = pos_embeds[0] + pos_embeds[1] + pos_embeds[2] + pos_embeds[3]
+
+        split_sizes = [int(h * w) for t, h, w in grid_thw_list]
+        if len(split_sizes) > 1:
+            split_indices = list(accumulate(split_sizes[:-1]))
+            patch_pos_embeds_split = mx.split(patch_pos_embeds, split_indices, axis=0)
+        else:
+            patch_pos_embeds_split = [patch_pos_embeds]
+
+        patch_pos_embeds_permute = []
+        merge_size = self.config.spatial_merge_size
+
+        for pos_embed, (t, h, w) in zip(patch_pos_embeds_split, grid_thw_list):
+            t, h, w = int(t), int(h), int(w)
+            feature_dim = pos_embed.shape[-1]
+            pos_embed = mx.tile(pos_embed, (t, 1))
+            pos_embed = pos_embed.reshape(t, h, w, feature_dim)
+            pos_embed = (
+                pos_embed.reshape(
+                    t,
+                    h // merge_size,
+                    merge_size,
+                    w // merge_size,
+                    merge_size,
+                    feature_dim,
+                )
+                .transpose(0, 1, 3, 2, 4, 5)
+                .reshape(-1, feature_dim)
+            )
+            patch_pos_embeds_permute.append(pos_embed)
+
+        patch_pos_embeds = mx.concatenate(patch_pos_embeds_permute)
+        return patch_pos_embeds
+
+    def __call__(
+        self,
+        hidden_states: mx.array,
+        grid_thw: mx.array,
+        **kwargs,
+    ) -> mx.array:
+
+        hidden_states = self.patch_embed(hidden_states)
+        pos_embeds = self.fast_pos_embed_interpolate(grid_thw)
+        hidden_states = hidden_states + pos_embeds
+        rotary_pos_emb = self.rot_pos_emb(grid_thw)
+
+        seq_len = hidden_states.shape[0]
+        hidden_states = hidden_states.reshape(seq_len, -1)
+        rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)
+
+        # Assuming grid_thw has shape (batch_size, 3)
+        batch_size = grid_thw.shape[0]
+
+        # Calculate cu_seqlens for each item in the batch
+        cu_seqlens = []
+        for i in range(batch_size):
+            seq_len = grid_thw[i, 1] * grid_thw[i, 2]
+            cu_seqlens.append(mx.repeat(seq_len, grid_thw[i, 0]))
+
+        # Concatenate the cu_seqlens for all items in the batch
+        cu_seqlens = mx.concatenate(cu_seqlens)
+
+        cu_seqlens = mx.cumsum(cu_seqlens.astype(mx.int32), axis=0)
+        cu_seqlens = mx.pad(cu_seqlens, (1, 0), mode="constant", constant_values=0)
+
+        deepstack_feature_lists = []
+        for layer_num, blk in enumerate(self.blocks):
+            hidden_states = blk(
+                hidden_states,
+                cu_seqlens=cu_seqlens,
+                rotary_pos_emb=rotary_pos_emb,
+            )
+            if layer_num in self.deepstack_visual_indexes:
+                deepstack_feature = self.deepstack_merger_list[
+                    self.deepstack_visual_indexes.index(layer_num)
+                ](hidden_states)
+                deepstack_feature_lists.append(deepstack_feature)
+
+        hidden_states = self.merger(hidden_states)
+
+        return hidden_states, deepstack_feature_lists
+
+    def sanitize(self, weights):
+        sanitized_weights = {}
+        for k, v in weights.items():
+            if "position_ids" in k:
+                # Remove unused position_ids
+                continue
+            elif "patch_embed.proj.weight" in k:
+                # PyTorch conv2d weight tensors have shape:
+                #   [out_channels, in_channels, kH, KW]
+                # MLX conv2d expects the weight be of shape:
+                #   [out_channels, kH, KW, in_channels]
+                if check_array_shape(v):
+                    sanitized_weights[k] = v
+                else:
+                    sanitized_weights[k] = v.transpose(0, 2, 3, 4, 1)
+            else:
+                sanitized_weights[k] = v
+
+        return sanitized_weights
diff --git a/mlx_vlm/utils.py b/mlx_vlm/utils.py
index 8e3f7737..33db7fa5 100644
--- a/mlx_vlm/utils.py
+++ b/mlx_vlm/utils.py
@@ -241,6 +241,9 @@ def get_class_predicate(p, m):
             class_predicate=get_class_predicate,
         )
 
+    if hasattr(model, "sanitize"):
+        weights = model.sanitize(weights)
+
     model.load_weights(list(weights.items()))
     if not lazy:
         mx.eval(model.parameters())