我用的这个镜像，DCU的驱动为rock-5.7.1-6.2.30-1.ky10.x86_64，使用DOCKER部署，运行到这一步之后出错了

root@localhost:/# cd /root/
root@localhost:# ls
DeepSeek-R1-Distill-Qwen-32B deepseek-webui group_data pictures private_data public_data start.sh vllm-output.log 启动器.ipynb
root@localhost:# bash start.sh
Started vllm with PID 3215
Checking if output.log has content...
Waiting for output to appear in vllm-output.log...
Waiting for output to appear in vllm-output.log...
Waiting for output to appear in vllm-output.log...
INFO 03-18 23:18:48 api_server.py:526] vLLM API server version 0.6.2
INFO 03-18 23:18:48 api_server.py:527] args: Namespace(host=None, port=8000, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_auto_tool_choice=False, tool_call_parser=None, model='DeepSeek-R1-Distill-Qwen-32B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='float16', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=None, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=8, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=False, disable_sliding_window=False, use_v2_block_manager=False, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=None, max_num_seqs=256, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=8192, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, merge_lora=False, lora_target_modules=None, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=False, scheduler_delay_factor=0.0, enable_chunked_prefill=None, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, num_speculative_heads=None, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', tree_style_spec_decoding=False, typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=None, qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, disable_log_requests=False, max_log_len=None, disable_fastapi_docs=False)
INFO 03-18 23:18:48 api_server.py:164] Multiprocessing frontend to use ipc:///tmp/d1adf692-a96d-4ecf-884b-e493958e9a22 for IPC Path.
INFO 03-18 23:18:48 api_server.py:177] Started engine process with PID 3294
WARNING 03-18 23:18:48 config.py:1687] Casting torch.bfloat16 to torch.float16.
INFO 03-18 23:18:48 config.py:908] Defaulting to use mp for distributed inference
INFO 03-18 23:18:48 config.py:937] Disabled the custom all-reduce kernel because it is not supported on hcus.
WARNING 03-18 23:18:48 arg_utils.py:953] Chunked prefill is enabled by default for models with max_model_len > 32K. Currently, chunked prefill might not work with some features or models. If you encounter any issues, please disable chunked prefill by setting --enable-chunked-prefill=False.
INFO 03-18 23:18:48 config.py:1019] Chunked prefill is enabled with max_num_batched_tokens=512.
WARNING 03-18 23:18:48 config.py:398] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used
WARNING 03-18 23:18:56 config.py:1687] Casting torch.bfloat16 to torch.float16.
INFO 03-18 23:18:56 config.py:908] Defaulting to use mp for distributed inference
INFO 03-18 23:18:56 config.py:937] Disabled the custom all-reduce kernel because it is not supported on hcus.
WARNING 03-18 23:18:56 arg_utils.py:953] Chunked prefill is enabled by default for models with max_model_len > 32K. Currently, chunked prefill might not work with some features or models. If you encounter any issues, please disable chunked prefill by setting --enable-chunked-prefill=False.
INFO 03-18 23:18:56 config.py:1019] Chunked prefill is enabled with max_num_batched_tokens=512.
WARNING 03-18 23:18:56 config.py:398] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used
INFO 03-18 23:18:56 llm_engine.py:226] Initializing an LLM engine (v0.6.2) with config: model='DeepSeek-R1-Distill-Qwen-32B', speculative_config=None, tokenizer='DeepSeek-R1-Distill-Qwen-32B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=131072, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=8, pipeline_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=DeepSeek-R1-Distill-Qwen-32B, use_v2_block_manager=False, num_scheduler_steps=1, multi_step_stream_outputs=False, enable_prefix_caching=False, use_async_output_proc=False, use_cached_outputs=True, mm_processor_kwargs=None)
WARNING 03-18 23:18:57 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
INFO 03-18 23:18:57 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager
INFO 03-18 23:18:57 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
(VllmWorkerProcess pid=3450) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3450) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.418285 3449 ProcessGroupNCCL.cpp:821] [PG 0 Rank 4] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 4, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=350682096
I0318 23:19:05.418829 3449 ProcessGroupNCCL.cpp:821] [PG 1 Rank 4] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 4, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x14e6fbf0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=356487104
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.475135 3450 ProcessGroupNCCL.cpp:821] [PG 0 Rank 5] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 5, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=382059680
I0318 23:19:05.475838 3450 ProcessGroupNCCL.cpp:821] [PG 1 Rank 5] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 5, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x16c5c4a0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=387888880
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:05 selector.py:121] Using ROCmFlashAttention backend.
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.565550 3452 ProcessGroupNCCL.cpp:821] [PG 0 Rank 7] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 7, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=350452336
I0318 23:19:05.566150 3452 ProcessGroupNCCL.cpp:821] [PG 1 Rank 7] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 7, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x14e37a70, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=356293664
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:05 multiproc_worker_utils.py:218] Worker ready; awaiting tasks
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.599623 3448 ProcessGroupNCCL.cpp:821] [PG 0 Rank 3] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 3, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=356789536
I0318 23:19:05.600183 3448 ProcessGroupNCCL.cpp:821] [PG 1 Rank 3] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 3, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x15442d20, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=362584320
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.615216 3451 ProcessGroupNCCL.cpp:821] [PG 0 Rank 6] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 6, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=356800208
I0318 23:19:05.615767 3451 ProcessGroupNCCL.cpp:821] [PG 1 Rank 6] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 6, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x154456d0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=362591104
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.615924 3446 ProcessGroupNCCL.cpp:821] [PG 0 Rank 1] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 1, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=377447568
I0318 23:19:05.616483 3446 ProcessGroupNCCL.cpp:821] [PG 1 Rank 1] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 1, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x167f6490, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=383262320
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.841795 3447 ProcessGroupNCCL.cpp:821] [PG 0 Rank 2] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 2, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=358629440
I0318 23:19:05.842460 3447 ProcessGroupNCCL.cpp:821] [PG 1 Rank 2] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 2, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x15604040, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=364466784
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0318 23:19:05.848421 3294 ProcessGroupNCCL.cpp:821] [PG 0 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 0, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=426787488
I0318 23:19:05.849002 3294 ProcessGroupNCCL.cpp:821] [PG 1 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 0, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x197042a0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=422540816
I0318 23:19:05.867909 3446 ProcessGroupNCCL.cpp:821] [PG 2 Rank 1] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 1, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x167f6490, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=383294800
I0318 23:19:05.869334 3447 ProcessGroupNCCL.cpp:821] [PG 2 Rank 2] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 2, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x15604040, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=364489760
I0318 23:19:05.870071 3294 ProcessGroupNCCL.cpp:821] [PG 2 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 0, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x197042a0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=426505280
I0318 23:19:05.875190 3450 ProcessGroupNCCL.cpp:821] [PG 2 Rank 5] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 5, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x16c5c4a0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=387902912
I0318 23:19:05.875851 3452 ProcessGroupNCCL.cpp:821] [PG 2 Rank 7] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 7, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x14e37a70, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=356303184
I0318 23:19:05.875828 3448 ProcessGroupNCCL.cpp:821] [PG 2 Rank 3] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 3, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x15442d20, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=362648560
I0318 23:19:05.876214 3451 ProcessGroupNCCL.cpp:821] [PG 2 Rank 6] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 6, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x154456d0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=363170016
I0318 23:19:05.878188 3449 ProcessGroupNCCL.cpp:821] [PG 2 Rank 4] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 8, global rank: 4, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0x14e6fbf0, SPLIT_COLOR: 4318754687966092895, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=356534384
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
(VllmWorkerProcess pid=3450) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3450) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:05 utils.py:996] Found nccl from library librccl.so.1
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:05 pynccl.py:63] vLLM is using nccl==2.18.3
INFO 03-18 23:19:07 shm_broadcast.py:241] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3, 4, 5, 6, 7], buffer=<vllm.distributed.device_communicators.shm_broadcast.ShmRingBuffer object at 0x7fb688431570>, local_subscribe_port=51563, remote_subscribe_port=None)
I0318 23:19:07.152997 3294 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 0, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=431885248
I0318 23:19:07.153244 3447 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 2, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=365700384
I0318 23:19:07.153259 3446 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 1, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=390231840
I0318 23:19:07.153514 3448 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 3, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=367805984
I0318 23:19:07.153527 3449 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 4, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=367935904
I0318 23:19:07.153812 3451 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 6, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=373576464
I0318 23:19:07.153825 3450 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 5, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=391705824
I0318 23:19:07.153935 3452 ProcessGroupNCCL.cpp:821] [PG 3 Rank 0] ProcessGroupNCCL initialization options: NCCL version: 2.18.3, size: 1, global rank: 7, TORCH_NCCL_ASYNC_ERROR_HANDLING: 3, TORCH_NCCL_DUMP_ON_TIMEOUT: 0, TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC: 60000, TORCH_NCCL_DESYNC_DEBUG: 0, TORCH_NCCL_ENABLE_TIMING: 0, TORCH_NCCL_BLOCKING_WAIT: 0, TIMEOUT(ms): 600000, USE_HIGH_PRIORITY_STREAM: 0, SPLIT_FROM: 0, SPLIT_COLOR: 0, TORCH_DISTRIBUTED_DEBUG: OFF, TORCH_NCCL_ENABLE_MONITORING: 1, TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: 600, TORCH_NCCL_TRACE_BUFFER_SIZE: 0, TORCH_NCCL_COORD_CHECK_MILSEC: 1000, ID=368253520
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3450) INFO 03-18 23:19:07 model_runner.py:1041] Starting to load model DeepSeek-R1-Distill-Qwen-32B...
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3450) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:07 selector.py:121] Using ROCmFlashAttention backend.
Loading safetensors checkpoint shards: 0% Completed | 0/8 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 12% Completed | 1/8 [00:01<00:07, 1.07s/it]
Loading safetensors checkpoint shards: 25% Completed | 2/8 [00:02<00:06, 1.12s/it]
Loading safetensors checkpoint shards: 38% Completed | 3/8 [00:03<00:05, 1.14s/it]
Loading safetensors checkpoint shards: 50% Completed | 4/8 [00:04<00:04, 1.11s/it]
Loading safetensors checkpoint shards: 62% Completed | 5/8 [00:05<00:03, 1.10s/it]
Loading safetensors checkpoint shards: 75% Completed | 6/8 [00:06<00:02, 1.10s/it]
Loading safetensors checkpoint shards: 88% Completed | 7/8 [00:07<00:01, 1.13s/it]
Loading safetensors checkpoint shards: 100% Completed | 8/8 [00:08<00:00, 1.07it/s]
Loading safetensors checkpoint shards: 100% Completed | 8/8 [00:08<00:00, 1.04s/it]

(VllmWorkerProcess pid=3450) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
(VllmWorkerProcess pid=3451) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
(VllmWorkerProcess pid=3446) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
(VllmWorkerProcess pid=3452) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
(VllmWorkerProcess pid=3449) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:16 model_runner.py:1052] Loading model weights took 7.7084 GB
I0318 23:19:17.358732 3294 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 0] ProcessGroupNCCL created ncclComm_ 0x1af2db00 on CUDA device:
I0318 23:19:17.358784 3294 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 0] NCCL_DEBUG: N/A
I0318 23:19:17.359401 3450 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 5] ProcessGroupNCCL created ncclComm_ 0x19127120 on CUDA device:
I0318 23:19:17.359448 3450 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 5] NCCL_DEBUG: N/A
I0318 23:19:17.360271 3449 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 4] ProcessGroupNCCL created ncclComm_ 0x1733a030 on CUDA device:
I0318 23:19:17.360323 3449 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 4] NCCL_DEBUG: N/A
I0318 23:19:17.360350 3451 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 6] ProcessGroupNCCL created ncclComm_ 0x16d76100 on CUDA device:
I0318 23:19:17.360360 3446 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 1] ProcessGroupNCCL created ncclComm_ 0x18cc2480 on CUDA device:
I0318 23:19:17.360349 3447 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 2] ProcessGroupNCCL created ncclComm_ 0x17ace340 on CUDA device:
I0318 23:19:17.360363 3448 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 3] ProcessGroupNCCL created ncclComm_ 0x1790e060 on CUDA device:
I0318 23:19:17.360394 3451 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 6] NCCL_DEBUG: N/A
I0318 23:19:17.360394 3446 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 1] NCCL_DEBUG: N/A
I0318 23:19:17.360404 3448 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 3] NCCL_DEBUG: N/A
I0318 23:19:17.360366 3452 ProcessGroupNCCL.cpp:1991] [PG 2 Rank 7] ProcessGroupNCCL created ncclComm_ 0x172f62d0 on CUDA device:
I0318 23:19:17.360404 3447 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 2] NCCL_DEBUG: N/A
I0318 23:19:17.360419 3452 ProcessGroupNCCL.cpp:1996] [PG 2 Rank 7] NCCL_DEBUG: N/A
Wind(VllmWorkerProcess pid=3448) INFO 03-18 23:19:20 model_runner_base.py:120] Writing input of failed execution to /tmp/err_execute_model_input_20250318-231920.pkl...
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:20 model_runner_base.py:120] Writing input of failed execution to /tmp/err_execute_model_input_20250318-231920.pkl...
(VllmWorkerProcess pid=3448) INFO 03-18 23:19:20 model_runner_base.py:149] Completed writing input of failed execution to /tmp/err_execute_model_input_20250318-231920.pkl.
(VllmWorkerProcess pid=3447) INFO 03-18 23:19:20 model_runner_base.py:149] Completed writing input of failed execution to /tmp/err_execute_model_input_20250318-231920.pkl.
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] Exception in worker VllmWorkerProcess while processing method determine_num_available_blocks: Error in model execution (input dumped to /tmp/err_execute_model_input_20250318-231920.pkl): HIP error: invalid device function
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] For debugging consider passing AMD_SERIALIZE_KERNEL=3.
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] Compile with TORCH_USE_HIP_DSA  to enable device-side assertions.
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] , Traceback (most recent call last):
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] File "/opt/conda/lib/python3.10/site-packages/vllm/worker/model_runner_base.py", line 116, in _wrapper
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] return func(*args, **kwargs)
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] File "/opt/conda/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1622, in execute_model
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] hidden_or_intermediate_states = model_executable(
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] File "/opt/conda/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 400, in forward
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] hidden_states = self.model(input_ids, positions, kv_caches,
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=3448) ERROR 03-18 23:19:20 multiproc_worker_utils.py:233] return self._call_impl(*args, **kwargs)