光合开发者社区-论坛

#启动命令 vllm serve ${MODEL_PATH} \ --port=${HTTP_PORT} \ --tensor-parallel-size=${WORLD_SIZE} \ --served-model-name Qwen2.5-7B-Instruct \ --enable-prefix-caching \ --gpu_memory_utilization ${GPU_MEMORY_UTILIZATION-0.80} \ --max-model-len 8192 \ --max-num-seqs 128 \ --enable-lora \ --lora-modules \ cg=${CG_MODEL_PATH}

# 错误日志 Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:07<00:21, 7.24s/it] Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:12<00:12, 6.33s/it] Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:20<00:06, 6.85s/it] Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:25<00:00, 6.11s/it] Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:25<00:00, 6.35s/it] ERROR 05-25 18:26:34 [engine.py:476] infer_schema(func): Parameter lora_b_weights has unsupported type list[torch.Tensor]. The valid types are: dict_keys([<class 'torch.Tensor'>, typing.Optional[torch.Tensor], typing.Sequence[torch.Tensor], typing.List[torch.Tensor], typing.Sequence[typing.Optional[torch.Tensor]], typing.List[typing.Optional[torch.Tensor]], <class 'int'>, typing.Optional[int], typing.Sequence[int], typing.List[int], typing.Optional[typing.Sequence[int]], typing.Optional[typing.List[int]], <class 'float'>, typing.Optional[float], typing.Sequence[float], typing.List[float], typing.Optional[typing.Sequence[float]], typing.Optional[typing.List[float]], <class 'bool'>, typing.Optional[bool], typing.Sequence[bool], typing.List[bool], typing.Optional[typing.Sequence[bool]], typing.Optional[typing.List[bool]], <class 'str'>, typing.Optional[str], typing.Union[int, float, bool], typing.Union[int, float, bool, NoneType], typing.Sequence[typing.Union[int, float, bool]], typing.List[typing.Union[int, float, bool]], <class 'torch.dtype'>, typing.Optional[torch.dtype], <class 'torch.device'>, typing.Optional[torch.device]]). Got func with signature (inputs: torch.Tensor, lora_b_weights: list[torch.Tensor], output_tensor: torch.Tensor, token_lora_mapping: torch.Tensor, token_indices_sorted_by_lora_ids: torch.Tensor, num_tokens_per_lora: torch.Tensor, lora_token_start_loc: torch.Tensor, lora_ids: torch.Tensor, no_lora_flag_cpu: torch.Tensor, offset_start: int = 0, add_inputs: bool = False) -> None) ERROR 05-25 18:26:34 [engine.py:476] Traceback (most recent call last): ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 464, in run_mp_engine ERROR 05-25 18:26:34 [engine.py:476] engine = MQLLMEngine.from_vllm_config( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 139, in from_vllm_config ERROR 05-25 18:26:34 [engine.py:476] return cls( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 93, in __init__ ERROR 05-25 18:26:34 [engine.py:476] self.engine = LLMEngine(*args, **kwargs) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py", line 273, in __init__ ERROR 05-25 18:26:34 [engine.py:476] self.model_executor = executor_class(vllm_config=vllm_config) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 53, in __init__ ERROR 05-25 18:26:34 [engine.py:476] self._init_executor() Process SpawnProcess-1: ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/executor/uniproc_executor.py", line 48, in _init_executor ERROR 05-25 18:26:34 [engine.py:476] self.collective_rpc("load_model") ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/executor/uniproc_executor.py", line 57, in collective_rpc ERROR 05-25 18:26:34 [engine.py:476] answer = run_method(self.driver_worker, method, args, kwargs) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2910, in run_method ERROR 05-25 18:26:34 [engine.py:476] return func(*args, **kwargs) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 211, in load_model ERROR 05-25 18:26:34 [engine.py:476] self.model_runner.load_model() ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1214, in load_model ERROR 05-25 18:26:34 [engine.py:476] self.model = self.lora_manager.create_lora_manager(self.model) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/worker_manager.py", line 204, in create_lora_manager ERROR 05-25 18:26:34 [engine.py:476] lora_manager = create_lora_manager( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/models.py", line 812, in create_lora_manager ERROR 05-25 18:26:34 [engine.py:476] lora_manager = lora_manager_cls( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/models.py", line 735, in __init__ ERROR 05-25 18:26:34 [engine.py:476] super().__init__(model, max_num_seqs, max_num_batched_tokens, ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/models.py", line 351, in __init__ ERROR 05-25 18:26:34 [engine.py:476] self.punica_wrapper = get_punica_wrapper( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/punica_wrapper/punica_selector.py", line 15, in get_punica_wrapper ERROR 05-25 18:26:34 [engine.py:476] punica_wrapper_cls = resolve_obj_by_qualname(punica_wrapper_qualname) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2478, in resolve_obj_by_qualname ERROR 05-25 18:26:34 [engine.py:476] module = importlib.import_module(module_name) ERROR 05-25 18:26:34 [engine.py:476] File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module ERROR 05-25 18:26:34 [engine.py:476] return _bootstrap._gcd_import(name[level:], package, level) ERROR 05-25 18:26:34 [engine.py:476] File "<frozen importlib._bootstrap>", line 1050, in _gcd_import ERROR 05-25 18:26:34 [engine.py:476] File "<frozen importlib._bootstrap>", line 1027, in _find_and_load ERROR 05-25 18:26:34 [engine.py:476] File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked ERROR 05-25 18:26:34 [engine.py:476] File "<frozen importlib._bootstrap>", line 688, in _load_unlocked ERROR 05-25 18:26:34 [engine.py:476] File "<frozen importlib._bootstrap_external>", line 883, in exec_module ERROR 05-25 18:26:34 [engine.py:476] File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/punica_wrapper/punica_gpu.py", line 19, in <module> ERROR 05-25 18:26:34 [engine.py:476] from vllm.lora.ops.triton_ops import (LoRAKernelMeta, lora_expand, ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/ops/triton_ops/__init__.py", line 4, in <module> ERROR 05-25 18:26:34 [engine.py:476] from vllm.lora.ops.triton_ops.lora_expand_op import lora_expand ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/lora/ops/triton_ops/lora_expand_op.py", line 281, in <module> ERROR 05-25 18:26:34 [engine.py:476] direct_register_custom_op( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2460, in direct_register_custom_op ERROR 05-25 18:26:34 [engine.py:476] schema_str = torch.library.infer_schema(op_func, ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/_library/infer_schema.py", line 106, in infer_schema ERROR 05-25 18:26:34 [engine.py:476] error_fn( ERROR 05-25 18:26:34 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/_library/infer_schema.py", line 58, in error_fn ERROR 05-25 18:26:34 [engine.py:476] raise ValueError( ERROR 05-25 18:26:34 [engine.py:476] ValueError: infer_schema(func): Parameter lora_b_weights has unsupported type list[torch.Tensor]. The valid types are: dict_keys([<class 'torch.Tensor'>, typing.Optional[torch.Tensor], typing.Sequence[torch.Tensor], typing.List[torch.Tensor], typing.Sequence[typing.Optional[torch.Tensor]], typing.List[typing.Optional[torch.Tensor]], <class 'int'>, typing.Optional[int], typing.Sequence[int], typing.List[int], typing.Optional[typing.Sequence[int]], typing.Optional[typing.List[int]], <class 'float'>, typing.Optional[float], typing.Sequence[float], typing.List[float], typing.Optional[typing.Sequence[float]], typing.Optional[typing.List[float]], <class 'bool'>, typing.Optional[bool], typing.Sequence[bool], typing.List[bool], typing.Optional[typing.Sequence[bool]], typing.Optional[typing.List[bool]], <class 'str'>, typing.Optional[str], typing.Union[int, float, bool], typing.Union[int, float, bool, NoneType], typing.Sequence[typing.Union[int, float, bool]], typing.List[typing.Union[int, float, bool]], <class 'torch.dtype'>, typing.Optional[torch.dtype], <class 'torch.device'>, typing.Optional[torch.device]]). Got func with signature (inputs: torch.Tensor, lora_b_weights: list[torch.Tensor], output_tensor: torch.Tensor, token_lora_mapping: torch.Tensor, token_indices_sorted_by_lora_ids: torch.Tensor, num_tokens_per_lora: torch.Tensor, lora_token_start_loc: torch.Tensor, lora_ids: torch.Tensor, no_lora_flag_cpu: torch.Tensor, offset_start: int = 0, add_inputs: bool = False) -> None) Traceback (most recent call last): File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap self.run() File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run self._target(*self._args, **self._kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 478, in run_mp_engine raise e from None File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 464, in run_mp_engine engine = MQLLMEngine.from_vllm_config( File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 139, in from_vllm_config return cls( File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 93, in __init__ self.engine = LLMEngine(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py", line 273, in __init__ self.model_executor = executor_class(vllm_config=vllm_config) File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 53, in __init__ self._init_executor() File "/usr/local/lib/python3.10/dist-packages/vllm/executor/uniproc_executor.py", line 48, in _init_executor self.collective_rpc("load_model") File "/usr/local/lib/python3.10/dist-packages/vllm/executor/uniproc_executor.py", line 57, in collective_rpc answer = run_method(self.driver_worker, method, args, kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2910, in run_method return func(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 211, in load_model self.model_runner.load_model() File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1214, in load_model self.model = self.lora_manager.create_lora_manager(self.model) File "/usr/local/lib/python3.10/dist-packages/vllm/lora/worker_manager.py", line 204, in create_lora_manager lora_manager = create_lora_manager( File "/usr/local/lib/python3.10/dist-packages/vllm/lora/models.py", line 812, in create_lora_manager lora_manager = lora_manager_cls( File "/usr/local/lib/python3.10/dist-packages/vllm/lora/models.py", line 735, in __init__ super().__init__(model, max_num_seqs, max_num_batched_tokens, File "/usr/local/lib/python3.10/dist-packages/vllm/lora/models.py", line 351, in __init__ self.punica_wrapper = get_punica_wrapper( File "/usr/local/lib/python3.10/dist-packages/vllm/lora/punica_wrapper/punica_selector.py", line 15, in get_punica_wrapper punica_wrapper_cls = resolve_obj_by_qualname(punica_wrapper_qualname) File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2478, in resolve_obj_by_qualname module = importlib.import_module(module_name) File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "<frozen importlib._bootstrap>", line 1050, in _gcd_import File "<frozen importlib._bootstrap>", line 1027, in _find_and_load File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked File "<frozen importlib._bootstrap>", line 688, in _load_unlocked File "<frozen importlib._bootstrap_external>", line 883, in exec_module File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed File "/usr/local/lib/python3.10/dist-packages/vllm/lora/punica_wrapper/punica_gpu.py", line 19, in <module> from vllm.lora.ops.triton_ops import (LoRAKernelMeta, lora_expand, File "/usr/local/lib/python3.10/dist-packages/vllm/lora/ops/triton_ops/__init__.py", line 4, in <module> from vllm.lora.ops.triton_ops.lora_expand_op import lora_expand File "/usr/local/lib/python3.10/dist-packages/vllm/lora/ops/triton_ops/lora_expand_op.py", line 281, in <module> direct_register_custom_op( File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2460, in direct_register_custom_op schema_str = torch.library.infer_schema(op_func, File "/usr/local/lib/python3.10/dist-packages/torch/_library/infer_schema.py", line 106, in infer_schema error_fn( File "/usr/local/lib/python3.10/dist-packages/torch/_library/infer_schema.py", line 58, in error_fn raise ValueError( ValueError: infer_schema(func): Parameter lora_b_weights has unsupported type list[torch.Tensor]. The valid types are: dict_keys([<class 'torch.Tensor'>, typing.Optional[torch.Tensor], typing.Sequence[torch.Tensor], typing.List[torch.Tensor], typing.Sequence[typing.Optional[torch.Tensor]], typing.List[typing.Optional[torch.Tensor]], <class 'int'>, typing.Optional[int], typing.Sequence[int], typing.List[int], typing.Optional[typing.Sequence[int]], typing.Optional[typing.List[int]], <class 'float'>, typing.Optional[float], typing.Sequence[float], typing.List[float], typing.Optional[typing.Sequence[float]], typing.Optional[typing.List[float]], <class 'bool'>, typing.Optional[bool], typing.Sequence[bool], typing.List[bool], typing.Optional[typing.Sequence[bool]], typing.Optional[typing.List[bool]], <class 'str'>, typing.Optional[str], typing.Union[int, float, bool], typing.Union[int, float, bool, NoneType], typing.Sequence[typing.Union[int, float, bool]], typing.List[typing.Union[int, float, bool]], <class 'torch.dtype'>, typing.Optional[torch.dtype], <class 'torch.device'>, typing.Optional[torch.device]]). Got func with signature (inputs: torch.Tensor, lora_b_weights: list[torch.Tensor], output_tensor: torch.Tensor, token_lora_mapping: torch.Tensor, token_indices_sorted_by_lora_ids: torch.Tensor, num_tokens_per_lora: torch.Tensor, lora_token_start_loc: torch.Tensor, lora_ids: torch.Tensor, no_lora_flag_cpu: torch.Tensor, offset_start: int = 0, add_inputs: bool = False) -> None) I0525 18:26:35.327848 562 ProcessGroupNCCL.cpp:1275] [PG ID 0 PG GUID 0 Rank 0] ProcessGroupNCCL destructor entered. W0525 18:26:35.327929 562 ProcessGroupNCCL.cpp:1279] Warning: WARNING: process group has NOT been destroyed before we destruct ProcessGroupNCCL. On normal program exit, the application should call destroy_process_group to ensure that any pending NCCL operations have finished in this process. In rare cases this process can exit before this point and block the progress of another member of the process group. This constraint has always been present, but this warning has only been added since PyTorch 2.4 (function operator()) I0525 18:26:35.327947 562 ProcessGroupNCCL.cpp:1259] [PG ID 0 PG GUID 0 Rank 0] Launching ProcessGroupNCCL abort asynchrounously. I0525 18:26:35.328169 562 ProcessGroupNCCL.cpp:1145] [PG ID 0 PG GUID 0 Rank 0] future is successfully executed for: ProcessGroup abort I0525 18:26:35.328181 562 ProcessGroupNCCL.cpp:1266] [PG ID 0 PG GUID 0 Rank 0] ProcessGroupNCCL aborts successfully. I0525 18:26:35.328197 562 ProcessGroupNCCL.cpp:1296] [PG ID 0 PG GUID 0 Rank 0] ProcessGroupNCCL watchdog thread joined. I0525 18:26:35.328226 562 ProcessGroupNCCL.cpp:1300] [PG ID 0 PG GUID 0 Rank 0] ProcessGroupNCCL heart beat monitor thread joined. Traceback (most recent call last): File "/usr/local/bin/vllm", line 7, in <module> sys.exit(main()) File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/cli/main.py", line 65, in main args.dispatch_function(args) File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/cli/serve.py", line 55, in cmd uvloop.run(run_server(args)) File "/usr/local/lib/python3.10/dist-packages/uvloop/__init__.py", line 69, in run return loop.run_until_complete(wrapper()) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete File "/usr/local/lib/python3.10/dist-packages/uvloop/__init__.py", line 48, in wrapper return await main File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/api_server.py", line 1431, in run_server await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/api_server.py", line 1451, in run_server_worker async with build_async_engine_client(args, client_config) as engine_client: File "/usr/lib/python3.10/contextlib.py", line 199, in __aenter__ return await anext(self.gen) File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/api_server.py", line 158, in build_async_engine_client async with build_async_engine_client_from_engine_args( File "/usr/lib/python3.10/contextlib.py", line 199, in __aenter__ return await anext(self.gen) File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/api_server.py", line 291, in build_async_engine_client_from_engine_args raise RuntimeError( RuntimeError: Engine process failed to start. See stack trace for the root cause.

解决了，因为vllm的类型与torch校验因支持不同python版本差异导致校验失败，将/usr/local/lib/python3.10/dist-packages/torch/_library/infer_schema.py中typing.List[typ]前增加list[typ], 已确认lora可以启动且推理正常