镜像： vllm:0.9.2-ubuntu22.04-dtk25.04.2-das1.7-py3.10-20251203 双节点启动ray后执行 vllm serve /data/model/Qwen3-32B -tp 8 --max-model-len 40000 --gpu-memory-utilization 0.97 --distributed-executor-backend ray --enforce-eager 报错： (RayWorkerWrapper pid=8508) (RayWorkerWrapper pid=25324, ip=11.2.3.12) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] Error executing method 'determine_num_available_blocks'. This might cause deadlock in distributed execution. (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] Traceback (most recent call last): (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 674, in execute_method (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return run_method(self, method, args, kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2910, in run_method (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 257, in determine_num_available_blocks (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] self.model_runner.profile_run() (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1315, in profile_run (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] self._dummy_run(max_num_batched_tokens, max_num_seqs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1441, in _dummy_run (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] self.execute_model(model_input, kv_caches, intermediate_tensors) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1881, in execute_model (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] hidden_or_intermediate_states = model_executable( (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 311, in forward (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] hidden_states = self.model(input_ids, positions, intermediate_tensors, (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/compilation/decorators.py", line 179, in __call__ (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return self.forward(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen2.py", line 379, in forward (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] hidden_states, residual = layer( (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 224, in forward (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] hidden_states = self.self_attn( (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 155, in forward (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] attn_output = self.attn(q, k, v) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 258, in forward (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] torch.ops.vllm.unified_attention_with_output( (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/_ops.py", line 1116, in __call__ (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] return self._op(*args, **(kwargs or {})) (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 508, in unified_attention_with_output (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] self.impl.forward(self, (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 851, in forward (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] self.sdpa_attn_func( (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 1088, in _sdpa_attention (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] sub_out = torch.nn.functional.scaled_dot_product_attention( (RayWorkerWrapper pid=25324, ip=11.2.3.12) ERROR 02-04 15:48:35 [worker_base.py:682] IndexError: max(): Expected reduction dim 2 to have non-zero size. (RayWorkerWrapper pid=25323, ip=11.2.3.12) /usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py:1088: UserWarning: 1Torch was not compiled with memory efficient attention. (Triggered internally at /home/pytorch/aten/src/ATen/native/transformers/hip/sdp_utils.cpp:663.) (RayWorkerWrapper pid=25323, ip=11.2.3.12) sub_out = torch.nn.functional.scaled_dot_product_attention( /usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py:1088: UserWarning: 1Torch was not compiled with memory efficient attention. (Triggered internally at /home/pytorch/aten/src/ATen/native/transformers/hip/sdp_utils.cpp:663.) sub_out = torch.nn.functional.scaled_dot_product_attention( ERROR 02-04 15:48:35 [worker_base.py:682] Error executing method 'determine_num_available_blocks'. This might cause deadlock in distributed execution. ERROR 02-04 15:48:35 [worker_base.py:682] Traceback (most recent call last): ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 674, in execute_method ERROR 02-04 15:48:35 [worker_base.py:682] return run_method(self, method, args, kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2910, in run_method ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 257, in determine_num_available_blocks ERROR 02-04 15:48:35 [worker_base.py:682] self.model_runner.profile_run() ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1315, in profile_run ERROR 02-04 15:48:35 [worker_base.py:682] self._dummy_run(max_num_batched_tokens, max_num_seqs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1441, in _dummy_run ERROR 02-04 15:48:35 [worker_base.py:682] self.execute_model(model_input, kv_caches, intermediate_tensors) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 02-04 15:48:35 [worker_base.py:682] return func(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1881, in execute_model ERROR 02-04 15:48:35 [worker_base.py:682] hidden_or_intermediate_states = model_executable( ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 311, in forward ERROR 02-04 15:48:35 [worker_base.py:682] hidden_states = self.model(input_ids, positions, intermediate_tensors, ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/compilation/decorators.py", line 179, in __call__ ERROR 02-04 15:48:35 [worker_base.py:682] return self.forward(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen2.py", line 379, in forward ERROR 02-04 15:48:35 [worker_base.py:682] hidden_states, residual = layer( ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 224, in forward ERROR 02-04 15:48:35 [worker_base.py:682] hidden_states = self.self_attn( ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 155, in forward ERROR 02-04 15:48:35 [worker_base.py:682] attn_output = self.attn(q, k, v) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [worker_base.py:682] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 258, in forward ERROR 02-04 15:48:35 [worker_base.py:682] torch.ops.vllm.unified_attention_with_output( ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/torch/_ops.py", line 1116, in __call__ ERROR 02-04 15:48:35 [worker_base.py:682] return self._op(*args, **(kwargs or {})) ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 508, in unified_attention_with_output ERROR 02-04 15:48:35 [worker_base.py:682] self.impl.forward(self, ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 851, in forward ERROR 02-04 15:48:35 [worker_base.py:682] self.sdpa_attn_func( ERROR 02-04 15:48:35 [worker_base.py:682] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 1088, in _sdpa_attention ERROR 02-04 15:48:35 [worker_base.py:682] sub_out = torch.nn.functional.scaled_dot_product_attention( ERROR 02-04 15:48:35 [worker_base.py:682] IndexError: max(): Expected reduction dim 2 to have non-zero size. ERROR 02-04 15:48:35 [engine.py:476] max(): Expected reduction dim 2 to have non-zero size. ERROR 02-04 15:48:35 [engine.py:476] Traceback (most recent call last): ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 464, in run_mp_engine ERROR 02-04 15:48:35 [engine.py:476] engine = MQLLMEngine.from_vllm_config( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 139, in from_vllm_config ERROR 02-04 15:48:35 [engine.py:476] return cls( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 93, in __init__ ERROR 02-04 15:48:35 [engine.py:476] self.engine = LLMEngine(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py", line 276, in __init__ ERROR 02-04 15:48:35 [engine.py:476] self._initialize_kv_caches() ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py", line 424, in _initialize_kv_caches ERROR 02-04 15:48:35 [engine.py:476] self.model_executor.determine_num_available_blocks()) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 104, in determine_num_available_blocks ERROR 02-04 15:48:35 [engine.py:476] results = self.collective_rpc("determine_num_available_blocks") ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 332, in collective_rpc ERROR 02-04 15:48:35 [engine.py:476] return self._run_workers(method, *args, **(kwargs or {})) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/executor/ray_distributed_executor.py", line 517, in _run_workers ERROR 02-04 15:48:35 [engine.py:476] self.driver_worker.execute_method(sent_method, *args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 683, in execute_method ERROR 02-04 15:48:35 [engine.py:476] raise e ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 674, in execute_method ERROR 02-04 15:48:35 [engine.py:476] return run_method(self, method, args, kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2910, in run_method ERROR 02-04 15:48:35 [engine.py:476] return func(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 02-04 15:48:35 [engine.py:476] return func(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 257, in determine_num_available_blocks ERROR 02-04 15:48:35 [engine.py:476] self.model_runner.profile_run() ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 02-04 15:48:35 [engine.py:476] return func(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1315, in profile_run ERROR 02-04 15:48:35 [engine.py:476] self._dummy_run(max_num_batched_tokens, max_num_seqs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1441, in _dummy_run ERROR 02-04 15:48:35 [engine.py:476] self.execute_model(model_input, kv_caches, intermediate_tensors) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 02-04 15:48:35 [engine.py:476] return func(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1881, in execute_model ERROR 02-04 15:48:35 [engine.py:476] hidden_or_intermediate_states = model_executable( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [engine.py:476] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [engine.py:476] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 311, in forward ERROR 02-04 15:48:35 [engine.py:476] hidden_states = self.model(input_ids, positions, intermediate_tensors, ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/compilation/decorators.py", line 179, in __call__ ERROR 02-04 15:48:35 [engine.py:476] return self.forward(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen2.py", line 379, in forward ERROR 02-04 15:48:35 [engine.py:476] hidden_states, residual = layer( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [engine.py:476] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [engine.py:476] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 224, in forward ERROR 02-04 15:48:35 [engine.py:476] hidden_states = self.self_attn( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [engine.py:476] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [engine.py:476] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 155, in forward ERROR 02-04 15:48:35 [engine.py:476] attn_output = self.attn(q, k, v) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 02-04 15:48:35 [engine.py:476] return self._call_impl(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 02-04 15:48:35 [engine.py:476] return forward_call(*args, **kwargs) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 258, in forward ERROR 02-04 15:48:35 [engine.py:476] torch.ops.vllm.unified_attention_with_output( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/torch/_ops.py", line 1116, in __call__ ERROR 02-04 15:48:35 [engine.py:476] return self._op(*args, **(kwargs or {})) ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 508, in unified_attention_with_output ERROR 02-04 15:48:35 [engine.py:476] self.impl.forward(self, ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 851, in forward ERROR 02-04 15:48:35 [engine.py:476] self.sdpa_attn_func( ERROR 02-04 15:48:35 [engine.py:476] File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 1088, in _sdpa_attention ERROR 02-04 15:48:35 [engine.py:476] sub_out = torch.nn.functional.scaled_dot_product_attention( ERROR 02-04 15:48:35 [engine.py:476] IndexError: max(): Expected reduction dim 2 to have non-zero size. Process SpawnProcess-1: Traceback (most recent call last): File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap self.run() File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run self._target(*self._args, **self._kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 478, in run_mp_engine raise e from None File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 464, in run_mp_engine engine = MQLLMEngine.from_vllm_config( File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 139, in from_vllm_config return cls( File "/usr/local/lib/python3.10/dist-packages/vllm/engine/multiprocessing/engine.py", line 93, in __init__ self.engine = LLMEngine(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py", line 276, in __init__ self._initialize_kv_caches() File "/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py", line 424, in _initialize_kv_caches self.model_executor.determine_num_available_blocks()) File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 104, in determine_num_available_blocks results = self.collective_rpc("determine_num_available_blocks") File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 332, in collective_rpc return self._run_workers(method, *args, **(kwargs or {})) File "/usr/local/lib/python3.10/dist-packages/vllm/executor/ray_distributed_executor.py", line 517, in _run_workers self.driver_worker.execute_method(sent_method, *args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 683, in execute_method raise e File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 674, in execute_method return run_method(self, method, args, kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/utils/__init__.py", line 2910, in run_method return func(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context return func(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 257, in determine_num_available_blocks self.model_runner.profile_run() File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context return func(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1315, in profile_run self._dummy_run(max_num_batched_tokens, max_num_seqs) File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1441, in _dummy_run self.execute_model(model_input, kv_caches, intermediate_tensors) File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context return func(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1881, in execute_model hidden_or_intermediate_states = model_executable( File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 311, in forward hidden_states = self.model(input_ids, positions, intermediate_tensors, File "/usr/local/lib/python3.10/dist-packages/vllm/compilation/decorators.py", line 179, in __call__ return self.forward(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen2.py", line 379, in forward hidden_states, residual = layer( File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 224, in forward hidden_states = self.self_attn( File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/qwen3.py", line 155, in forward attn_output = self.attn(q, k, v) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 258, in forward torch.ops.vllm.unified_attention_with_output( File "/usr/local/lib/python3.10/dist-packages/torch/_ops.py", line 1116, in __call__ return self._op(*args, **(kwargs or {})) File "/usr/local/lib/python3.10/dist-packages/vllm/attention/layer.py", line 508, in unified_attention_with_output self.impl.forward(self, File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 851, in forward self.sdpa_attn_func( File "/usr/local/lib/python3.10/dist-packages/vllm/attention/backends/rocm_flash_attn.py", line 1088, in _sdpa_attention sub_out = torch.nn.functional.scaled_dot_product_attention( IndexError: max(): Expected reduction dim 2 to have non-zero size.