@@ -170,15 +170,15 @@ cleanup() {
170
170
# Function to start router
171
171
start_router () {
172
172
local log_file=" $LOG_DIR /router.log"
173
-
173
+
174
174
print_status " Starting router with round-robin routing (stress test mode)"
175
-
175
+
176
176
# Create log directory
177
177
mkdir -p " $( dirname " $log_file " ) "
178
-
178
+
179
179
# Set stress test mode
180
180
export VLLM_ROUTER_STRESS_TEST_MODE=true
181
-
181
+
182
182
# Start router with detailed logging
183
183
python3 -m src.vllm_router.app --port " $ROUTER_PORT " \
184
184
--service-discovery static \
@@ -188,10 +188,10 @@ start_router() {
188
188
--routing-logic roundrobin \
189
189
--log-stats \
190
190
--log-stats-interval 5 > " $log_file " 2>&1 &
191
-
191
+
192
192
ROUTER_PID=$!
193
193
print_status " Router started with PID: $ROUTER_PID "
194
-
194
+
195
195
# Wait for router to be ready
196
196
print_status " Waiting for router to be ready..."
197
197
timeout 30 bash -c " until curl -s http://localhost:$ROUTER_PORT /v1/models > /dev/null 2>&1; do sleep 1; done" || {
@@ -207,7 +207,7 @@ start_router() {
207
207
run_stress_test () {
208
208
print_status " Running stress test with Apache Bench"
209
209
print_status " Concurrent: $CONCURRENT , Total: $REQUESTS "
210
-
210
+
211
211
# Create payload file
212
212
local payload_file=" /tmp/stress_payload.json"
213
213
cat > " $payload_file " << EOF
@@ -220,7 +220,7 @@ run_stress_test() {
220
220
"temperature": 0.7
221
221
}
222
222
EOF
223
-
223
+
224
224
# Run Apache Bench
225
225
ab -c " $CONCURRENT " \
226
226
-n " $REQUESTS " \
@@ -229,53 +229,55 @@ EOF
229
229
-H " Authorization: Bearer test" \
230
230
-H " x-user-id: stress-test-user" \
231
231
" http://localhost:$ROUTER_PORT /v1/chat/completions"
232
-
232
+
233
233
# Clean up payload file
234
234
rm -f " $payload_file "
235
-
235
+
236
236
print_status " Stress test completed"
237
-
237
+
238
238
# Small delay to ensure all logs are written
239
239
sleep 2
240
240
}
241
241
242
242
# Function to check round-robin correctness
243
243
check_roundrobin_correctness () {
244
244
local log_file=" $LOG_DIR /router.log"
245
-
245
+
246
246
print_status " Checking round-robin routing correctness..."
247
-
247
+
248
248
if [ ! -f " $log_file " ]; then
249
249
print_error " Router log file not found: $log_file "
250
250
return 1
251
251
fi
252
-
252
+
253
253
# Extract backend routing decisions from logs
254
254
# Look for "Routing request ... to http://localhost:XXXX"
255
- local backend1_count=$( grep -c " to http://localhost:$BACKEND1_PORT " " $log_file " || echo " 0" )
256
- local backend2_count=$( grep -c " to http://localhost:$BACKEND2_PORT " " $log_file " || echo " 0" )
255
+ local backend1_count
256
+ backend1_count=$( grep -c " to http://localhost:$BACKEND1_PORT " " $log_file " || echo " 0" )
257
+ local backend2_count
258
+ backend2_count=$( grep -c " to http://localhost:$BACKEND2_PORT " " $log_file " || echo " 0" )
257
259
local total_routed=$(( backend1 _count + backend2 _count))
258
-
260
+
259
261
print_status " Round-robin routing results:"
260
262
print_status " Backend localhost:$BACKEND1_PORT : $backend1_count requests"
261
263
print_status " Backend localhost:$BACKEND2_PORT : $backend2_count requests"
262
264
print_status " Total routed: $total_routed requests"
263
-
265
+
264
266
if [ " $total_routed " -eq 0 ]; then
265
267
print_error " No routing decisions found in logs"
266
268
return 1
267
269
fi
268
-
270
+
269
271
# Calculate percentages
270
272
local backend1_pct=$(( backend1 _count * 100 / total_routed))
271
273
local backend2_pct=$(( backend2 _count * 100 / total_routed))
272
-
274
+
273
275
print_status " Backend localhost:$BACKEND1_PORT : ${backend1_pct} %"
274
276
print_status " Backend localhost:$BACKEND2_PORT : ${backend2_pct} %"
275
-
277
+
276
278
# Check if distribution is roughly even (within 20% tolerance)
277
279
local diff=$(( backend1 _pct > backend2 _pct ? backend1 _pct - backend2 _pct : backend2 _pct - backend1 _pct))
278
-
280
+
279
281
if [ " $diff " -le 20 ]; then
280
282
print_status " ✅ Round-robin routing is working correctly (${diff} % difference)"
281
283
return 0
@@ -290,7 +292,7 @@ check_roundrobin_correctness() {
290
292
# Function to show log summary
291
293
show_log_summary () {
292
294
local log_file=" $LOG_DIR /router.log"
293
-
295
+
294
296
if [ -f " $log_file " ]; then
295
297
print_status " Log summary (last 20 lines):"
296
298
tail -20 " $log_file " | sed ' s/^/ /'
368
370
print_error " Test completed but round-robin routing correctness check failed!"
369
371
show_log_summary
370
372
exit 1
371
- fi
373
+ fi
0 commit comments