无法使用flash_attn 2.6.1+das.opt1.dtk25042.20260115.g0f919271。 来自于这个pytorch镜像 ```shell docker pull harbor.sourcefind.cn:5443/dcu/admin/base/pytorch:2.5.1-ubuntu22.04-dtk25.04.4-1230-py3.10-20260115 ``` 代码如下： ```python print("\n[Check flash_attn kernel]") try: from flash_attn.flash_attn_interface import flash_attn_func q = torch.randn(1, 4, 8, 64, device="cuda", dtype=dtype) k = torch.randn(1, 4, 8, 64, device="cuda", dtype=dtype) v = torch.randn(1, 4, 8, 64, device="cuda", dtype=dtype) o = flash_attn_func(q, k, v, dropout_p=0.0, causal=True) print("flash_attn_func OK, output shape:", o.shape) flash_kernel_ok = True except Exception as e: print("flash_attn_func FAILED:", e) flash_kernel_ok = False ``` 报错如下 ```shell cqu_cs_201@b516fc2b3cdb:~/project/vllm-demo$ python test_flash_attn.py Torch: 2.5.1 CUDA available: True CUDA device: K100 [Check flash_attn kernel] flash_attn_func FAILED: HIP error: invalid device function HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing AMD_SERIALIZE_KERNEL=3 Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. [Load model with FlashAttention2] `torch_dtype` is deprecated! Use `dtype` instead! Attention module types: {'Qwen2Attention', 'Linear'} Traceback (most recent call last): File "/public/home/cqu_cs_201/project/vllm-demo/test_flash_attn.py", line 58, in _ = model(**inputs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/utils/generic.py", line 918, in wrapper output = func(self, *args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 449, in forward outputs: BaseModelOutputWithPast = self.model( File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/utils/generic.py", line 1072, in wrapper outputs = func(self, *args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 384, in forward hidden_states = decoder_layer( File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/modeling_layers.py", line 94, in __call__ return super().__call__(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func return func(*args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 234, in forward hidden_states, _ = self.self_attn( File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func return func(*args, **kwargs) File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 169, in forward attn_output, attn_weights = attention_interface( File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/integrations/flash_attention.py", line 66, in flash_attention_forward attn_output = _flash_attention_forward( File "/public/home/cqu_cs_201/.local/lib/python3.10/site-packages/transformers/modeling_flash_attention_utils.py", line 664, in _flash_attention_forward out = flash_fn(query_states, key_states, value_states, **flash_kwargs) File "/usr/local/lib/python3.10/dist-packages/flash_attn/flash_attn_interface.py", line 934, in flash_attn_func return FlashAttnFunc.apply( File "/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py", line 575, in apply return super().apply(*args, **kwargs) # type: ignore[misc] File "/usr/local/lib/python3.10/dist-packages/flash_attn/flash_attn_interface.py", line 570, in forward out, q, k, v, out_padded, softmax_lse, S_dmask, rng_state = _flash_attn_forward( File "/usr/local/lib/python3.10/dist-packages/flash_attn/flash_attn_interface.py", line 52, in _flash_attn_forward out, q, k, v, out_padded, softmax_lse, S_dmask, rng_state, skip_info = flash_attn_cuda.fwd( RuntimeError: HIP error: invalid device function HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing AMD_SERIALIZE_KERNEL=3 Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. ``` 同时也导致使用vllm,transformer时无法使用flash-attn