Skip to content

Fix ExLlamaV2 kernels on Windows: link cublas.lib and enable PLATFORM.WIN32 #2944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Qubitium merged 1 commit into from
Jul 9, 2026
Merged

Fix ExLlamaV2 kernels on Windows: link cublas.lib and enable PLATFORM.WIN32 #2944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion gptqmodel/nn_modules/qlinear/exllamav2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class ExllamaV2Linear(GPTQQuantLinear):
SUPPORTS_OUT_FEATURES_DIVISIBLE_BY = [32]

SUPPORTS_DEVICES = [DEVICE.CUDA] # ROCm has broken accuracies issues
SUPPORTS_PLATFORM = [PLATFORM.LINUX]
SUPPORTS_PLATFORM = [PLATFORM.LINUX, PLATFORM.WIN32]
SUPPORTS_PACK_DTYPES = [torch.int32]
SUPPORTS_ADAPTERS = [Lora]

Expand Down
2 changes: 1 addition & 1 deletion gptqmodel/nn_modules/qlinear/exllamav2_awq.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class AwqExllamaV2Linear(AWQuantLinear):
SUPPORTS_OUT_FEATURES_DIVISIBLE_BY = [32]

SUPPORTS_DEVICES = [DEVICE.CUDA] # ROCm has broken accuracies issues
SUPPORTS_PLATFORM = [PLATFORM.LINUX]
SUPPORTS_PLATFORM = [PLATFORM.LINUX, PLATFORM.WIN32]
SUPPORTS_PACK_DTYPES = [torch.int32]
SUPPORTS_ADAPTERS = [Lora]

Expand Down
17 changes: 17 additions & 0 deletions gptqmodel/utils/exllamav2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

from __future__ import annotations

import os
import sys
from pathlib import Path

import torch
Expand Down Expand Up @@ -93,6 +95,19 @@ def _exllamav2_extra_cuda_cflags() -> list[str]:
)


def _exllamav2_extra_ldflags() -> list[str]:
# q_gemm.cu / q_gemm_awq.cu call cublasHgemm. On Linux the symbol resolves
# at load time, but the MSVC linker needs the import library explicitly or
# the build fails with `LNK2019: unresolved external symbol cublasHgemm`.
# Mirrors _extra_ldflags in gptqmodel/exllamav3/ext.py.
if sys.platform != "win32":
return []
flags = ["cublas.lib"]
if sys.base_prefix != sys.prefix:
flags.append(f"/LIBPATH:{os.path.join(sys.base_prefix, 'libs')}")
return flags


def _exllamav2_awq_extra_cflags() -> list[str]:
return default_jit_cflags(opt_level=None, enable_bf16=True)

Expand Down Expand Up @@ -120,6 +135,7 @@ def _exllamav2_awq_extra_cuda_cflags() -> list[str]:
extra_cflags=_exllamav2_gptq_extra_cflags,
extra_cuda_cflags=_exllamav2_gptq_extra_cuda_cflags,
extra_include_paths=_exllamav2_include_paths,
extra_ldflags=_exllamav2_extra_ldflags,
force_rebuild_env="GPTQMODEL_EXLLAMAV2_FORCE_REBUILD",
verbose_env="GPTQMODEL_EXT_VERBOSE",
requires_cuda=True,
Expand All @@ -142,6 +158,7 @@ def _exllamav2_awq_extra_cuda_cflags() -> list[str]:
extra_cflags=_exllamav2_awq_extra_cflags,
extra_cuda_cflags=_exllamav2_awq_extra_cuda_cflags,
extra_include_paths=_exllamav2_include_paths,
extra_ldflags=_exllamav2_extra_ldflags,
force_rebuild_env="GPTQMODEL_EXLLAMAV2_AWQ_FORCE_REBUILD",
verbose_env="GPTQMODEL_EXT_VERBOSE",
requires_cuda=True,
Expand Down