# Qwen3.5-122B-A10B vLLM 多机多卡部署文档 ## 一、环境说明 | 项目 | 节点1 | 节点2 | |------|-------|-------| | IP | 10.1.1.1 | 10.1.1.2 | | GPU | 8x DCU | 8x DCU | | NUMA | 0-7 | 8-15 | | 网络 | bond1 | bond1 | | 总 GPU | 16 张 | TP=16 | - **镜像**: `harbor.sourcefind.cn:5443/dcu/admin/base/custom:vllm018-ubuntu22.04-dtk26.04-gemma4-0413` - **vLLM**: 0.18.1 | **模型**: Qwen3.5-122B-A10B | **框架**: Ray --- ## 二、Docker Compose 启动前看卡在不hy-smi  ```yaml version: "3.8" services: vllm: image: harbor.sourcefind.cn:5443/dcu/admin/base/custom:vllm018-ubuntu22.04-dtk26.04-gemma4-0413 container_name: vllm stdin_open: true tty: true shm_size: "200" network_mode: host privileged: true devices: - /dev/kfd - /dev/dri - /dev/mkfd group_add: - video cap_add: - SYS_PTRACE security_opt: - seccomp=unconfined user: root volumes: - /opt/hyhal/:/opt/hyhal/:ro - /data/models:/models command: bash ``` 进容器: `docker exec -it vllm /bin/bash` --- ## 三、环境变量 查看网卡速率 ethtool bond1 | grep -E "Speed|Duplex" ethtool bond0 | grep -E "Speed|Duplex" ethtool ens2f0np0 | grep -E "Speed|Duplex"   ### 节点1 ```bash export ALLREDUCE_STREAM_WITH_COMPUTE=1 export VLLM_HOST_IP=10.1.1.1 export NCCL_SOCKET_IFNAME=ens2f0np0 export GLOO_SOCKET_IFNAME=ens2f0np0 unset NCCL_ALGO export NCCL_MIN_NCHANNELS=16 export NCCL_MAX_NCHANNELS=16 export NCCL_NET_GDR_READ=0 export NCCL_NET_GDR_LEVEL=0 export NCCL_IB_DISABLE=1 export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export VLLM_RPC_TIMEOUT=1800000 export VLLM_USE_V1=0 export VLLM_NUMA_BIND=1 export VLLM_RANK0_NUMA=0 export VLLM_RANK1_NUMA=1 export VLLM_RANK2_NUMA=2 export VLLM_RANK3_NUMA=3 export VLLM_RANK4_NUMA=4 export VLLM_RANK5_NUMA=5 export VLLM_RANK6_NUMA=6 export VLLM_RANK7_NUMA=7 ``` ### 节点2 ```bash export ALLREDUCE_STREAM_WITH_COMPUTE=1 export VLLM_HOST_IP=10.1.1.2 export NCCL_SOCKET_IFNAME=ens2f0np0 export GLOO_SOCKET_IFNAME=ens2f0np0 unset NCCL_ALGO export NCCL_MIN_NCHANNELS=16 export NCCL_MAX_NCHANNELS=16 export NCCL_NET_GDR_READ=0 export NCCL_NET_GDR_LEVEL=0 export NCCL_IB_DISABLE=1 export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export VLLM_RPC_TIMEOUT=1800000 export VLLM_USE_V1=0 export VLLM_NUMA_BIND=1 export VLLM_RANK0_NUMA=8 export VLLM_RANK1_NUMA=9 export VLLM_RANK2_NUMA=10 export VLLM_RANK3_NUMA=11 export VLLM_RANK4_NUMA=12 export VLLM_RANK5_NUMA=13 export VLLM_RANK6_NUMA=14 export VLLM_RANK7_NUMA=15 ``` > **⚠️ 节点2 HIP_VISIBLE_DEVICES=0-7，不是8-15。rocm-smi验证。** --- ## 四、Ray 集群 ```bash # 节点1 ray start --head --node-ip-address=10.1.1.1 --port=6379 --num-gpus=8 --num-cpus=32 # 节点2 ray start --address='10.1.1.1:6379' --num-gpus=8 --num-cpus=32 # 验证 ray status ray list nodes echo $HIP_VISIBLE_DEVICES # 清理 #ray stop #pkill -f vllm #pkill -f ray ``` --- ## 五、启动 vLLM ```bash export VLLM_USE_V1=0 vllm serve /models/Qwen3.5-122B-A10B \ --host 0.0.0.0 \ --port 1025 \ --served-model-name Qwen3.5-122B-A10B \ --tensor-parallel-size 16 \ --max-model-len 128000 \ --gpu-memory-utilization 0.9 \ --distributed-executor-backend ray \ >> ./vllm_serve.log 2>&1 & tail -f vllm_serve.log # 验证 curl localhost:1025/v1/models ```  --- ## 六、测试脚本 ### test.py（流式输出） ```python #!/usr/bin/env python3 # -*- coding: utf-8 -*- import time import json import requests URL = "http://localhost:1025/v1/chat/completions" MODEL = "Qwen3.5-122B-A10B" print("=== vLLM 流式性能测试 ===") while True: prompt = input("\n你：") if prompt.lower() in ("exit", "quit"): break payload = { "model": MODEL, "messages": [{"role": "user", "content": prompt}], "max_tokens": 3000, "temperature": 0.7, "stream": True } start_time = time.time() first_token_time = None char_count = 0 print("模型回答：", end="", flush=True) with requests.post(URL, json=payload, stream=True, timeout=300) as r: r.raise_for_status() for raw_line in r.iter_lines(): if not raw_line: continue line = raw_line.decode("utf-8").strip() if not line.startswith("data:"): continue data = line[5:].strip() if data == "[DONE]": break try: obj = json.loads(data) delta = obj["choices"][0]["delta"] content = ( delta.get("content") or delta.get("reasoning_content") or "" ) if content: if first_token_time is None: first_token_time = time.time() - start_time print(content, end="", flush=True) char_count += len(content) except Exception: continue total_time = time.time() - start_time est_tokens = int(char_count / 1.4) if char_count else 0 tps = est_tokens / total_time if total_time > 0 else 0 print("\n----------------------------------------") if first_token_time is not None: print(f"首字延迟 TTFT : {first_token_time:.3f} 秒") else: print("首字延迟 TTFT : 未收到响应") print(f"响应字符数 : {char_count}") print(f"估算 Token 数 : {est_tokens}") print(f"总耗时 : {total_time:.3f} 秒") print(f"生成速度 TPS : {tps:.2f} tokens/s") print("----------------------------------------") ```  --- ### test-context.py（长上下文测试） ```python #!/usr/bin/env python3 import time, json, requests URL = "http://localhost:1025/v1/chat/completions" MODEL = "Qwen3.5-122B-A10B" START_TOKENS = 20480 STEP_TOKENS = 20480 MAX_ATTEMPTS = 100 BASE_TEXT = ( "人工智能是计算机科学的一个分支，它企图了解智能的实质..." ) def generate_long_text(target_tokens): target_chars = int(target_tokens * 1.5) repeat_times = max(1, target_chars // len(BASE_TEXT)) return BASE_TEXT * repeat_times print("=== vLLM 最大上下文窗口测试 ===") current_tokens = START_TOKENS successful_tokens = 0 for attempt in range(MAX_ATTEMPTS): print(f"[测试 #{attempt+1}] {current_tokens} tokens...") long_prompt = generate_long_text(current_tokens) payload = { "model": MODEL, "messages": [ {"role": "system", "content": "你是一个有用的助手。"}, {"role": "user", "content": long_prompt + " 请总结上文核心观点。"} ], "max_tokens": 10, "temperature": 0.0, "stream": False } start = time.time() try: r = requests.post(URL, json=payload, timeout=120) elapsed = time.time() - start if r.status_code == 400: print(f"❌ 触发上下文限制。上限: {successful_tokens}") break r.raise_for_status() successful_tokens = current_tokens print(f"✅ 成功！耗时: {elapsed:.2f}s") current_tokens += STEP_TOKENS time.sleep(1) except Exception as e: print(f"❌ 错误: {e}") break print(f"最终结果: 支持的最大输入 Token 数约为 {successful_tokens}") ```  ### test-all.py（并发压测） ```python #!/usr/bin/env python3 import time, json, statistics, threading, requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry URL = "http://localhost:1025/v1/chat/completions" MODEL = "Qwen3.5-122B-A10B" ROUNDS = 3 CONCURRENT = 264 MAX_TOKENS = 1024 TEMPERATURE = 0.7 PROMPT = "请用非常详细的篇幅介绍人工智能的发展历史..." ttft_list, tps_list = [], [] success_cnt = 0 lock = threading.Lock() def get_session(): s = requests.Session() adapter = HTTPAdapter(pool_connections=CONCURRENT, pool_maxsize=CONCURRENT, max_retries=Retry(total=0)) s.mount("http://", adapter) return s def worker(tid, session): global success_cnt for i in range(ROUNDS): payload = { "model": MODEL, "messages": [{"role": "user", "content": PROMPT}], "max_tokens": MAX_TOKENS, "temperature": TEMPERATURE, "stream": True } start = time.time() ttft, chars = None, 0 try: with session.post(URL, json=payload, stream=True, timeout=600) as r: r.raise_for_status() for line in r.iter_lines(chunk_size=512): if not line or not line.startswith(b"data:"): continue data = line[5:].strip() if data == b"[DONE]": break obj = json.loads(data) content = obj["choices"][0]["delta"].get("content") or "" if content: if ttft is None: ttft = time.time() - start chars += len(content) except Exception as e: print(f"[线程 {tid}] 出错: {e}") continue total = time.time() - start est_tokens = int(chars / 1.4) tps = est_tokens / total if total > 0 else 0 with lock: if ttft and est_tokens > 0: ttft_list.append(ttft) tps_list.append(tps) success_cnt += 1 print(f"[线程{tid} 第{i+1}轮] TTFT:{ttft:.3f}s TPS:{tps:.2f}") print(f"=== 并发压测 (并发={CONCURRENT}, 轮次={ROUNDS}) ===") session = get_session() threads = [threading.Thread(target=worker, args=(tid+1, session)) for tid in range(CONCURRENT)] for t in threads: t.start() for t in threads: t.join() print(" =========== 压测汇总 ===========") print(f"成功请求: {success_cnt}") if ttft_list: print(f"TTFT 均值: {statistics.mean(ttft_list):.3f}s 最小:{min(ttft_list):.3f}s 最大:{max(ttft_list):.3f}s") print(f"TPS 均值: {statistics.mean(tps_list):.2f} 最小:{min(tps_list):.2f} 最大:{max(tps_list):.2f}") ``` ---