Skip to content

Commit 51fd10d

Browse files
Your NameHarmonyHu
Your Name
authored and
HarmonyHu
committed
support qwen2-audio
1 parent 4d798b6 commit 51fd10d

17 files changed

Lines changed: 488298 additions & 0 deletions

‎models/Qwen2_Audio/README.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
## Qwen2-Audio
2+
本工程实现BM1684X上的Qwen2-Audio部署多模态大模型Qwen2-Audio。通过TPU-MLIR将模型转换为BM1684X上的bmodel，实现高效的推理。并采用c++代码将其部署到BM1684X上，提供python接口调用。目前仅实现了SoC环境。
3+
4+
## 开发环境准备
5+
从Huggingface或者modelscope上下载Qwen2-Audio模型Qwen2-Audio-7B-Instruct文件。
6+
7+
## 编译模型
8+
9+
此处介绍如何将onnx模型编译成bmodel。可以直接下载编译好的模型。
10+
```
11+
# 1684X
12+
python3 -m dfss --url=open@sophgo.com:/ext_model_information/LLM/LLM-TPU/qwen2-audio-7b_w8f16_seq599_1dev.bmodel
13+
14+
```
15+
### 下载docker，启动容器
16+
```
17+
docker pull sophgo/tpuc_dev:latest
18+
docker run -it --rm --privileged --net=host --ipc=host -v $(pwd):/workspace sophgo/tpuc_dev:latest
19+
docker exec -it $(docker ps -lq) /bin/bash
20+
```
21+
### 下载TPU-MLIR代码并编译
22+
```
23+
git clone https://github.com/sophgo/tpu-mlir.git
24+
cd tpu-mlir
25+
git submodule update --init --recursive
26+
mkdir build && cd build
27+
cmake .. -DLLVM_TARGETS_TO_BUILD="BPF;X86" -DCMAKE_BUILD_TYPE=Release
28+
make -j$(nproc)
29+
```
30+
### 导出onnx模型
31+
将tools/文件里面的model_qwen2.py替换transformers里面的有关qwen2的相关文件。然后使用运行：
32+
```
33+
python3 export_onnx.py
34+
```
35+
导出模型，有些模型不适合导出onnx，仅需要导出pt文件，详情件export_onnx.py文件。
36+
### 编译生成bmodel
37+
将模型放置在合适的目录下，然后运行compile.sh脚本。
38+
```
39+
./compile.sh
40+
```
41+
## 编译与运行程序
42+
编译库文件，生成chat.cpython*.so文件，将该文件拷贝到pipeline.py文件目录; 并将bmodel文件和config目录拷贝过去
43+
```
44+
cd python_demo
45+
mkdir build
46+
cd build && cmake .. && make && cp *cpython* .. && cd ..
47+
python demo
48+
python3 pipeline.py -m qwen2-audio-7b_w8f16_seq599_1dev.bmodel -c config
49+
```

‎models/Qwen2_Audio/compile/compile.sh‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
exe_dir=$(dirname $(readlink -f "$0"))
5+
pushd $exe_dir
6+
7+
combined_outdir=../models/BM1684X/
8+
9+
if [ ! -d $combined_outdir ];
10+
then
11+
mkdir -p $combined_outdir
12+
else
13+
echo dir $combined_outdir exist
14+
fi
15+
echo $exe_dir
16+
models=
17+
folder=${exe_dir}"/../../models/onnx"
18+
device_args=""
19+
quantize_args="--quantize W8F16"
20+
addr_args="--addr_mode io_alone"
21+
name="qwen2-audio-7b"
22+
num_layers=
23+
out_model=$name.bmodel
24+
seq_length=599
25+
hidden_size=4096
26+
mode="int8"
27+
num_device=1
28+
audio_seq_length=128
29+
30+
while [[ $# -gt 0 ]]; do
31+
key="$1"
32+
33+
case $key in
34+
--mode)
35+
mode="$2"
36+
shift 2
37+
;;
38+
--name)
39+
name="$2"
40+
shift 2
41+
;;
42+
--addr_mode)
43+
addr_mode="$2"
44+
shift 2
45+
;;
46+
--seq_length)
47+
seq_length="$2"
48+
shift 2
49+
;;
50+
--audio_seq_length)
51+
audio_seq_length="$2"
52+
shift 2
53+
;;
54+
--dynamic)
55+
dynamic="$2"
56+
shift 2
57+
;;
58+
*)
59+
echo "Invalid option: $key" >&2
60+
exit 1
61+
;;
62+
:)
63+
echo "Option -$OPTARG requires an argument." >&2
64+
exit 1
65+
;;
66+
esac
67+
done
68+
69+
if [[ -z "$seq_length" ]]; then
70+
echo "Error: --seq_length is required." >&2
71+
exit 1
72+
fi
73+
74+
if [ "$name" = "qwen2-audio-7b" ]; then
75+
num_layers=32
76+
hidden_size=4096
77+
echo "Compile Qwen2-AUDIO-7B"
78+
elif [ "$name" = "qwen2-audio-2b" ]; then
79+
num_layers=28
80+
hidden_size=1536
81+
echo "Compile Qwen2-Audio-2B"
82+
else
83+
>&2 echo -e "Error: Invalid name $name, the input name must be \033[31mqwen2-vl-2b|qwen2-vl-7b\033[0m"
84+
exit 1
85+
fi
86+
87+
if [ x$mode == x"int8" ]; then
88+
quantize_args="--quantize W8F16"
89+
elif [ x$mode == x"bf16" ]; then
90+
quantize_args="--quantize BF16"
91+
elif [ x$mode == x"fp16" ]; then
92+
quantize_args="--quantize F16"
93+
elif [ x$mode == x"int4" ]; then
94+
quantize_args="--quantize W4BF16 --q_group_size 32"
95+
else
96+
echo "Error, unknown quantize mode"
97+
exit 1
98+
fi
99+
100+
timestamp=$(date "+%Y%m%d_%H%M%S")
101+
out_model=${name}_${mode}_seq${seq_length}_1dev_${timestamp}.bmodel
102+
103+
104+
if [ x$addr_mode == x"io_alone" ]; then
105+
addr_args="--addr_mode io_alone"
106+
fi
107+
108+
outdir=${folder}/$mode"_1dev"/embedding
109+
mkdir -p $outdir
110+
pushd $outdir
111+
112+
model_transform.py \
113+
--model_name embedding \
114+
--model_def ${exe_dir}/../../llm/embedding.onnx \
115+
--input_shapes [[1,${seq_length}]] \
116+
--mlir embedding.mlir
117+
118+
model_deploy.py \
119+
--mlir embedding.mlir \
120+
--quant_input \
121+
$quantize_args \
122+
--quant_output \
123+
--chip bm1684x \
124+
$device_args \
125+
$dyn_args \
126+
--model embedding.bmodel
127+
128+
model_transform.py \
129+
--model_name embedding_cache \
130+
--model_def ${exe_dir}/../../llm/embedding_cache.onnx \
131+
--input_shapes [[1,1]] \
132+
--mlir embedding_cache.mlir
133+
134+
model_deploy.py \
135+
--mlir embedding_cache.mlir \
136+
--quant_input \
137+
$quantize_args \
138+
--quant_output \
139+
--chip bm1684x \
140+
--quantize W8F16 \
141+
$device_args \
142+
--model embedding_cache.bmodel
143+
144+
models=$models' '$outdir'/embedding.bmodel '$outdir'/embedding_cache.bmodel '
145+
146+
rm -f *.npz
147+
popd
148+
echo $models
149+
150+
outdir=${folder}/$mode"_1dev"/lm_head
151+
mkdir -p $outdir
152+
pushd $outdir
153+
154+
model_transform.py \
155+
--model_name lm_head \
156+
--model_def ${exe_dir}/../../llm/lm_head.pt \
157+
--input_shapes [[1,1,4096]] \
158+
--mlir lm_head.mlir
159+
160+
model_deploy.py \
161+
--mlir lm_head.mlir \
162+
$quantize_args \
163+
--quant_input \
164+
--chip bm1684x \
165+
$device_args \
166+
--model lm_head.bmodel
167+
168+
models=$models' '$outdir'/lm_head.bmodel '
169+
rm -f *.npz
170+
popd
171+
echo $models
172+
173+
outdir=${folder}/$mode"_1dev"/block
174+
mkdir -p $outdir
175+
echo $outdir
176+
pushd $outdir
177+
178+
for ((i=0; i<$num_layers; i++)); do
179+
180+
model_transform.py \
181+
--model_name block_$i \
182+
--model_def ${exe_dir}/../../llm/block_$i.onnx \
183+
--input_shapes [[1,$seq_length,4096],[1,$seq_length],[1,$seq_length]] \
184+
--mlir block_$i.mlir
185+
186+
model_deploy.py \
187+
--mlir block_$i.mlir \
188+
$quantize_args \
189+
--quant_input \
190+
--quant_output \
191+
--chip bm1684x \
192+
$device_args \
193+
$dyn_args \
194+
--model block_$i.bmodel
195+
196+
model_transform.py \
197+
--model_name block_cache_$i \
198+
--model_def ${exe_dir}/../../llm/block_cache_$i.pt \
199+
--input_shapes [[1,1,4096],[1,1],[1,1,1,$seq_length],[1,32,$seq_length,$audio_seq_length],[1,32,$seq_length,$audio_seq_length]] \
200+
--mlir block_cache_$i.mlir
201+
202+
model_deploy.py \
203+
--mlir block_cache_$i.mlir \
204+
$quantize_args \
205+
--quant_input \
206+
--quant_output \
207+
--chip bm1684x \
208+
$device_args \
209+
$addr_args \
210+
$dyn_args \
211+
--model block_cache_$i.bmodel
212+
213+
rm -f *.npz
214+
215+
models=${models}${outdir}'/block_'$i'.bmodel '$outdir'/block_cache_'$i'.bmodel '
216+
217+
done
218+
popd
219+
echo $models
220+
221+
# Compile AUDIO model
222+
outdir=${folder}/$mode"_1dev"/audio
223+
mkdir -p $outdir
224+
pushd $outdir
225+
model_transform.py \
226+
--model_name audio \
227+
--model_def ${exe_dir}/../../llm/audio_ext_model.onnx \
228+
--input_shapes [[1,${audio_seq_length},3000],[1,1,1500,1500]] \
229+
--mlir audio.mlir
230+
231+
model_deploy.py \
232+
--mlir audio.mlir \
233+
--chip bm1684x \
234+
$quantize_args \
235+
--model audio.bmodel
236+
237+
popd
238+
models=$models' '$outdir'/audio.bmodel '
239+
240+
# Compile projector model
241+
outdir=${folder}/$mode"_1dev"/projector
242+
mkdir -p $outdir
243+
pushd $outdir
244+
model_transform.py \
245+
--model_name projector \
246+
--model_def ${exe_dir}/../../llm/multi_modal_projector.onnx \
247+
--input_shapes [[1,750,1280]] \
248+
--mlir projector.mlir
249+
250+
model_deploy.py \
251+
--mlir projector.mlir \
252+
--chip bm1684x \
253+
$quantize_args \
254+
--model projector.bmodel
255+
256+
popd
257+
models=$models' '$outdir'/projector.bmodel '
258+
259+
260+
# Compile greed model
261+
outdir=${folder}/$mode"_1dev"/greed
262+
mkdir -p $outdir
263+
pushd $outdir
264+
model_transform.py \
265+
--model_name greed \
266+
--model_def ${exe_dir}/../../llm/greed.pt \
267+
--input_shapes [[1,1,115630]] \
268+
--mlir greed.mlir
269+
270+
model_deploy.py \
271+
--mlir greed.mlir \
272+
--chip bm1684x \
273+
$quantize_args \
274+
--model greed.bmodel
275+
276+
popd
277+
models=$models' '$outdir'/greed.bmodel '
278+
279+
model_tool --combine $models -o $out_model
280+
chmod 666 $out_model
281+
mv $out_model $combined_outdir

‎models/Qwen2_Audio/python_demo/CMakeLists.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
cmake_minimum_required(VERSION 3.10)
2+
project(Qwen2-Audio)
3+
4+
if (NOT DEFINED TARGET_ARCH)
5+
set(TARGET_ARCH soc)
6+
endif()
7+
8+
include_directories(${PROJECT_SOURCE_DIR}/include)
9+
10+
include_directories(/opt/sophon/libsophon-current/include)
11+
link_directories(/opt/sophon/libsophon-current/lib)
12+
13+
add_definitions(-DDEBUG --std=c++17 -fPIC -Wall -Werror)
14+
set(CMAKE_BUILD_TYPE "Debug")
15+
16+
find_package(Python 3.10 REQUIRED COMPONENTS Interpreter Development)
17+
set(pybind11_DIR "$ENV{HOME}/.local/lib/python3.10/site-packages/pybind11/share/cmake/pybind11")
18+
find_package(pybind11 REQUIRED CONFIG)
19+
20+
pybind11_add_module(chat chat.cpp)
21+
target_link_libraries(chat PUBLIC bmrt bmlib)
22+
install(TARGETS chat DESTINATION python)
23+

0 commit comments

Comments
 (0)