FEAT: Auto ngl for llama.cpp backend (#3518)

Author: codingl2k1

.github/workflows/python.yaml

@@ -125,7 +125,7 @@ jobs:
             sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           fi
           pip install -e ".[dev]"
-          pip install "xllamacpp>=0.1.16"
+          pip install "xllamacpp>=0.1.18" gguf
           if [ "$MODULE" == "metal" ]; then
             conda install -c conda-forge "ffmpeg<7"
             pip install "mlx>=0.22.0"
@@ -167,6 +167,7 @@ jobs:
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U -e ".[audio]"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U "openai>1"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U modelscope
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U gguf
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U sse_starlette
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U xoscar
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U "python-jose[cryptography]"

setup.cfg

@@ -94,7 +94,8 @@ intel =
     torch==2.1.0a0
     intel_extension_for_pytorch==2.1.10+xpu
 llama_cpp =
-    xllamacpp>=0.1.16
+    xllamacpp>=0.1.18
+    gguf
 transformers =
     transformers>=4.46.0
     torch

xinference/deploy/docker/Dockerfile

@@ -40,7 +40,7 @@ RUN pip install --upgrade -i "$PIP_INDEX" pip setuptools wheel&& \
     git restore . && \
     pip install -i "$PIP_INDEX" --no-deps "." && \
     pip uninstall xllamacpp -y && \
-    pip install "xllamacpp>=0.1.16" --index-url https://xorbitsai.github.io/xllamacpp/whl/cu124 && \
+    pip install "xllamacpp>=0.1.18" --index-url https://xorbitsai.github.io/xllamacpp/whl/cu124 && \
     # clean packages
     pip cache purge
 

xinference/deploy/docker/cpu.Dockerfile

@@ -27,7 +27,7 @@ RUN python -m pip install --upgrade -i "$PIP_INDEX" pip && \
     python setup.py build_web && \
     git restore . && \
     pip install -i "$PIP_INDEX" --no-deps "." && \
-    pip install -i "$PIP_INDEX" "xllamacpp>=0.1.16" && \
+    pip install -i "$PIP_INDEX" "xllamacpp>=0.1.18" && \
     # clean packages
     pip cache purge
 

xinference/deploy/docker/requirements-base.txt

@@ -22,3 +22,4 @@ async-timeout
 peft
 opencv-contrib-python-headless
 setproctitle
+gguf

xinference/deploy/docker/requirements_cpu-base.txt

@@ -28,3 +28,4 @@ ormsgpack  # For Fish Speech
 cachetools  # For Fish Speech
 imageio-ffmpeg  # For video
 opencv-contrib-python-headless
+gguf

xinference/model/audio/tests/test_cosyvoice.py

@@ -19,6 +19,7 @@
 
 
 @pytest.mark.parametrize("model_name", ["CosyVoice-300M-SFT", "CosyVoice2-0.5B"])
+@pytest.mark.skip(reason="The diffusers on the GPU CI action is not compatible.")
 def test_cosyvoice_sft(setup, model_name):
     endpoint, _ = setup
     from ....client import Client
@@ -72,6 +73,7 @@ def test_cosyvoice_sft(setup, model_name):
 
 
 @pytest.mark.parametrize("model_name", ["CosyVoice-300M", "CosyVoice2-0.5B"])
+@pytest.mark.skip(reason="The diffusers on the GPU CI action is not compatible.")
 def test_cosyvoice(setup, model_name):
     endpoint, _ = setup
     from ....client import Client
@@ -122,6 +124,7 @@ def test_cosyvoice(setup, model_name):
 
 
 @pytest.mark.parametrize("model_name", ["CosyVoice-300M-Instruct", "CosyVoice2-0.5B"])
+@pytest.mark.skip(reason="The diffusers on the GPU CI action is not compatible.")
 def test_cosyvoice_instruct(setup, model_name):
     endpoint, _ = setup
     from ....client import Client
@@ -154,7 +157,8 @@ def test_cosyvoice_instruct(setup, model_name):
     else:
         # inference without instruction
         response = model.speech(
-            "在面对挑战时，他展现了非凡的<strong>勇气</strong>与<strong>智慧</strong>。", voice="中文男"
+            "在面对挑战时，他展现了非凡的<strong>勇气</strong>与<strong>智慧</strong>。",
+            voice="中文男",
         )
         assert type(response) is bytes
         assert len(response) > 0

xinference/model/llm/llama_cpp/core.py

@@ -15,6 +15,7 @@
 import importlib.util
 import logging
 import os
+import pprint
 import queue
 from typing import Iterator, List, Optional, Union
 
@@ -24,6 +25,7 @@
 from ..core import LLM
 from ..llm_family import LLMFamilyV1, LLMSpecV1
 from ..utils import ChatModelMixin
+from .memory import estimate_gpu_layers
 
 logger = logging.getLogger(__name__)
 
@@ -95,7 +97,12 @@ def match_json(
 
     def load(self):
         try:
-            from xllamacpp import CommonParams, Server
+            from xllamacpp import (
+                CommonParams,
+                Server,
+                get_device_info,
+                ggml_backend_dev_type,
+            )
         except ImportError:
             error_message = "Failed to import module 'xllamacpp'"
             installation_guide = ["Please make sure 'xllamacpp' is installed. "]
@@ -175,6 +182,41 @@ def load(self):
                 # Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.
                 # 0x7FFFFFFF is INT32 max, will be auto set to all layers
                 params.n_gpu_layers = 0x7FFFFFFF
+                try:
+                    device_info = get_device_info()
+                    gpus = [
+                        info
+                        for info in device_info
+                        if info["type"]
+                        == ggml_backend_dev_type.GGML_BACKEND_DEVICE_TYPE_GPU
+                    ]
+                    if gpus:
+                        logger.info(
+                            "Try to estimate num gpu layers, n_ctx: %s, n_batch: %s, n_parallel: %s, gpus:\n%s",
+                            params.n_ctx,
+                            params.n_batch,
+                            params.n_parallel,
+                            pprint.pformat(gpus),
+                        )
+                        estimate = estimate_gpu_layers(
+                            gpus=gpus,
+                            model_path=model_path,
+                            projectors=[mmproj] if mmproj else [],
+                            context_length=params.n_ctx,
+                            batch_size=params.n_batch,
+                            num_parallel=params.n_parallel,
+                            kv_cache_type="",
+                        )
+                        logger.info("Estimate num gpu layers: %s", estimate)
+                        if estimate.tensor_split:
+                            params.tensor_split = estimate.tensor_split
+                        else:
+                            params.n_gpu_layers = estimate.layers
+                except Exception as e:
+                    logger.exception(
+                        "Estimate num gpu layers for llama.cpp backend failed: %s", e
+                    )
+
             self._llm = Server(params)
             self._executor = concurrent.futures.ThreadPoolExecutor(
                 max_workers=max(10, n_threads)