1
3
1
DeepSeek
vllm推理使用guided_json报错
2025-07-21 13:48:52
106

使用的镜像是image.sourcefind.cn:5000/dcu/admin/base/custom:ray-2.42.1-ubuntu22.04-dtk25.04-rc6-das1.4-vllm0.7.2-py3.10-rocblas-86461，启动命令是:vllm serve /data/deepseek-r1-distill-qwen-32b --tensor-parallel-size 2 --max-model-len 32768 --trust-remote-code --port 8000 --host 0.0.0.0 --served-model-name deepseek-r1-distill-qwen-32b --enable-auto-tool-choice --tool-call-parser hermes,
启动后调用:

# 初始化OpenAI客户端
client = OpenAI(
    # 如果没有配置环境变量，请用百炼API Key替换：api_key="sk-xxx"
    api_key="ok",
    base_url="http://localhost:8000/v1"
)

class Peopel(BaseModel):
    name: str
    age: int
json_schema = Peopel.model_json_schema()


for i in range(20):
    # 创建聊天完成请求
    completion = client.chat.completions.create(
        model="deepseek-r1-distill-qwen-32b",  # 此处以 qvq-max 为例，可按需更换模型名称
        messages=[
            {'role': 'system', 'content': 'You are a helpful assistant.'},
            {'role': 'user', 'content': '你是谁？'}],
        stream=True,
        max_tokens=1024,
        max_completion_tokens=1024,
        extra_body={"guided_json": json_schema},
        # 解除以下注释会在最后一个chunk返回Token使用量
        stream_options={
            "include_usage": True
        }
    )

    print(str(i) + "\n" + "=" * 20 + "思考过程" + "=" * 20 + "\n")

    for chunk in completion:
        print(chunk)

出现错误:
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 2164, in _run_ninja_build
ERROR 07-21 09:53:31 serving_chat.py:661] subprocess.run(
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/subprocess.py", line 526, in run
ERROR 07-21 09:53:31 serving_chat.py:661] raise CalledProcessError(retcode, process.args,
ERROR 07-21 09:53:31 serving_chat.py:661] subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.
ERROR 07-21 09:53:31 serving_chat.py:661]
ERROR 07-21 09:53:31 serving_chat.py:661] The above exception was the direct cause of the following exception:
ERROR 07-21 09:53:31 serving_chat.py:661]
ERROR 07-21 09:53:31 serving_chat.py:661] Traceback (most recent call last):
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_chat.py", line 359, in chat_completion_stream_generator
ERROR 07-21 09:53:31 serving_chat.py:661] async for res in result_generator:
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/vllm/engine/multiprocessing/client.py", line 606, in _process_request
ERROR 07-21 09:53:31 serving_chat.py:661] params = await
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 553, in build_guided_decoding_logits_processor_async
ERROR 07-21 09:53:31 serving_chat.py:661] processor = await get_guided_decoding_logits_processor(
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/vllm/model_executor/guided_decoding/init .py", line 107, in get_guided_decoding_logits_processor
ERROR 07-21 09:53:31 serving_chat.py:661] from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/vllm/model_executor/guided_decoding/xgrammar_decoding.py", line 15, in <module> 
ERROR 07-21 09:53:31 serving_chat.py:661] import xgrammar as xgr
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/xgrammar/init .py", line 1, in <module> 
ERROR 07-21 09:53:31 serving_chat.py:661] from . import testing
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/xgrammar/testing.py", line 11, in <module> 
ERROR 07-21 09:53:31 serving_chat.py:661] from .matcher import GrammarMatcher, bitmask_dtype
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/xgrammar/matcher.py", line 13, in <module> 
ERROR 07-21 09:53:31 serving_chat.py:661] from .kernels import apply_token_bitmask_inplace_kernels
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/xgrammar/kernels/init .py", line 12, in <module> 
ERROR 07-21 09:53:31 serving_chat.py:661] from .apply_token_bitmask_inplace_cuda import apply_token_bitmask_inplace_cuda
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/xgrammar/kernels/apply_token_bitmask_inplace_cuda.py", line 54, in <module> 
ERROR 07-21 09:53:31 serving_chat.py:661] _load_torch_ops()
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/xgrammar/kernels/apply_token_bitmask_inplace_cuda.py", line 42, in _load_torch_ops
ERROR 07-21 09:53:31 serving_chat.py:661] torch.utils.cpp_extension.load_inline(
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1706, in load_inline
ERROR 07-21 09:53:31 serving_chat.py:661] return _jit_compile(
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1781, in _jit_compile
ERROR 07-21 09:53:31 serving_chat.py:661] _write_ninja_file_and_build_library(
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1893, in _write_ninja_file_and_build_library
ERROR 07-21 09:53:31 serving_chat.py:661] _run_ninja_build(
ERROR 07-21 09:53:31 serving_chat.py:661] File "/usr/local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 2180, in _run_ninja_build
ERROR 07-21 09:53:31 serving_chat.py:661] raise RuntimeError(message) from e
ERROR 07-21 09:53:31 serving_chat.py:661] RuntimeError: Error building extension 'xgrammar': [1/3] /opt/dtk/bin/hipcc -DWITH_HIP -DTORCH_EXTENSION_NAME=xgrammar -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1014" -isystem /usr/local/lib/python3.10/site-packages/torch/include -isystem /usr/local/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /usr/local/lib/python3.10/site-packages/torch/include/TH -isystem /usr/local/lib/python3.10/site-packages/torch/include/THC -isystem /usr/local/lib/python3.10/site-packages/torch/include/THH -isystem /opt/dtk/include -isystem /usr/local/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=1 -fPIC -std=c++17 -O3 -Wno-switch-bool -fPIC -D__HIP_PLATFORM_AMD__=1 -DUSE_ROCM=1 -DHIPBLAS_V2 -DCUDA_HAS_FP16=1 -D__HIP_NO_HALF_OPERATORS__=1 -D__HIP_NO_HALF_CONVERSIONS__=1 -O3 -std=c++17 --threads 4 -use_fast_math --offload-arch=gfx906 --offload-arch=gfx926 --offload-arch=gfx928 --offload-arch=gfx936 -fno-gpu-rdc -c /root/.cache/torch_extensions/py310_cpu/xgrammar/cuda.hip -o cuda.cuda.o
ERROR 07-21 09:53:31 serving_chat.py:661] FAILED: cuda.cuda.o
ERROR 07-21 09:53:31 serving_chat.py:661] /opt/dtk/bin/hipcc -DWITH_HIP -DTORCH_EXTENSION_NAME=xgrammar -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1014" -isystem /usr/local/lib/python3.10/site-packages/torch/include -isystem /usr/local/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /usr/local/lib/python3.10/site-packages/torch/include/TH -isystem /usr/local/lib/python3.10/site-packages/torch/include/THC -isystem /usr/local/lib/python3.10/site-packages/torch/include/THH -isystem /opt/dtk/include -isystem /usr/local/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=1 -fPIC -std=c++17 -O3 -Wno-switch-bool -fPIC -D__HIP_PLATFORM_AMD__=1 -DUSE_ROCM=1 -DHIPBLAS_V2 -DCUDA_HAS_FP16=1 -D__HIP_NO_HALF_OPERATORS__=1 -D__HIP_NO_HALF_CONVERSIONS__=1 -O3 -std=c++17 --threads 4 -use_fast_math --offload-arch=gfx906 --offload-arch=gfx926 --offload-arch=gfx928 --offload-arch=gfx936 -fno-gpu-rdc -c /root/.cache/torch_extensions/py310_cpu/xgrammar/cuda.hip -o cuda.cuda.o
ERROR 07-21 09:53:31 serving_chat.py:661] dcc: error: unsupported option '--threads'; did you mean '-mthreads'?
ERROR 07-21 09:53:31 serving_chat.py:661] dcc: error: no such file or directory: '4'
ERROR 07-21 09:53:31 serving_chat.py:661] [2/3] c++ -MMD -MF main.o.d -DTORCH_EXTENSION_NAME=xgrammar -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1014" -isystem /usr/local/lib/python3.10/site-packages/torch/include -isystem /usr/local/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /usr/local/lib/python3.10/site-packages/torch/include/TH -isystem /usr/local/lib/python3.10/site-packages/torch/include/THC -isystem /usr/local/lib/python3.10/site-packages/torch/include/THH -isystem /opt/dtk/include -isystem /usr/local/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=1 -fPIC -std=c++17 -O3 -Wno-switch-bool -c /root/.cache/torch_extensions/py310_cpu/xgrammar/main.cpp -o main.o -fPIC -D__HIP_PLATFORM_AMD__=1 -DUSE_ROCM=1 -DHIPBLAS_V2
ERROR 07-21 09:53:31 serving_chat.py:661] ninja: build stopped: subcommand failed.

这是什么原因呢

全部评论（10）
只看楼主
W
wangwei
1楼
看报错怎么感觉有cuda相关的问题？source /opt/dtk/cuda/env.sh 一下再试试？
2025/7/21
网
网友vQLBnh
回复 @wangwei

进入容器执行source /opt/dtk/cuda/env.sh 后，再执行vllm启动命令出现，RuntimeError: Only one platform plugin can be activated, but got: ['cuda', 'rocm'] 这个错误，

之前这个是正常请求没问题，但是加上extra_body={"guided_json": json_schema}这样的参数有报错了
2025/7/21
W
wangwei
回复 @wangwei
能发一下完整的测试命令？这边复现一下看看
2025/7/21
网
网友vQLBnh
回复 @wangwei

好的，谢谢！

启动命令:vllm serve /data/deepseek-r1-distill-qwen-32b --tensor-parallel-size 2 --max-model-len 32768 --trust-remote-code --port 8000 --host 0.0.0.0 --served-model-name deepseek-r1-distill-qwen-32b --enable-auto-tool-choice --tool-call-parser hermes

测试命令:

def test_openapi_guide():

from openai import OpenAI

import os

from pydantic import BaseModel

# 初始化OpenAI客户端

client = OpenAI(

# 如果没有配置环境变量，请用百炼API Key替换：api_key="sk-xxx"

api_key="ok",

base_url="http://localhost:8000/v1"

)

class Peopel(BaseModel):

name: str

age: int

json_schema = Peopel.model_json_schema()

for i in range(20):

# 创建聊天完成请求

completion = client.chat.completions.create(

model="deepseek-r1-distill-qwen-32b",

messages=[

{'role': 'system', 'content': 'You are a helpful assistant.'},

{'role': 'user', 'content': '你是谁？'}],

stream=True,

max_tokens=1024,

max_completion_tokens=1024,

extra_body={"guided_json": json_schema},

stream_options={

"include_usage": True

}

)

print(str(i) + "\n" + "=" * 20 + "思考过程" + "=" * 20 + "\n")

for chunk in completion:

print(chunk)
2025/7/21
全部9条回复
没有更多内容了
--
楼主
Level.1
0 帖子
楼主热帖
暂无数据
常见问题新帖
暂无数据