orker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] WorkerProc failed to start. (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] Traceback (most recent call last): (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/v1/executor/multiproc\_executor.py", line 894, in worker\_main (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] worker = WorkerProc(\*args, \*\*kwargs) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/v1/executor/multiproc\_executor.py", line 663, in \*\*init\*\* (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.worker.load\_model() (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/worker.py", line 443, in load\_model (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.model\_runner.load\_model(load\_dummy\_weights=load\_dummy\_weights) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/model\_runner.py", line 5349, in load\_model (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] raise e (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/model\_runner.py", line 5268, in load\_model (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.model = model\_loader.load\_model( (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/model\_loader/base\_loader.py", line 80, in load\_model (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] process\_weights\_after\_loading(model, model\_config, target\_device) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/model\_loader/utils.py", line 108, in process\_weights\_after\_loading (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] quant\_method.process\_weights\_after\_loading(module) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/layers/quantization/compressed\_tensors/compressed\_tensors\_moe/compressed\_tensors\_moe\_w8a8\_int8.py", line 154, in process\_weights\_after\_loading (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.moe\_quant\_config = self.get\_fused\_moe\_quant\_config(layer) (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/layers/quantization/compressed\_tensors/compressed\_tensors\_moe/compressed\_tensors\_moe\_w8a8\_int8.py", line 166, in get\_fused\_moe\_quant\_config (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return make\_int8\_moe\_quant\_config( (Worker\_TP6 pid=865) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] TypeError: install\_fl\_w8a8\_moe\_selector..make\_int8\_moe\_quant\_config\_fl() got an unexpected keyword argument 'int8\_backend' (EngineCore pid=555) INFO 09-28 03:10:39 [multiproc\_executor.py:455] [shutdown] Executor: waiting for worker exit count=8 (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] WorkerProc failed to start. (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] Traceback (most recent call last): (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/v1/executor/multiproc\_executor.py", line 894, in worker\_main (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] worker = WorkerProc(\*args, \*\*kwargs) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/v1/executor/multiproc\_executor.py", line 663, in \*\*init\*\* (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.worker.load\_model() (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/worker.py", line 443, in load\_model (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.model\_runner.load\_model(load\_dummy\_weights=load\_dummy\_weights) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/model\_runner.py", line 5349, in load\_model (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] raise e (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/model\_runner.py", line 5268, in load\_model (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.model = model\_loader.load\_model( (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/model\_loader/base\_loader.py", line 80, in load\_model (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] process\_weights\_after\_loading(model, model\_config, target\_device) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/model\_loader/utils.py", line 108, in process\_weights\_after\_loading (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] quant\_method.process\_weights\_after\_loading(module) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/layers/quantization/compressed\_tensors/compressed\_tensors\_moe/compressed\_tensors\_moe\_w8a8\_int8.py", line 154, in process\_weights\_after\_loading (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.moe\_quant\_config = self.get\_fused\_moe\_quant\_config(layer) (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/layers/quantization/compressed\_tensors/compressed\_tensors\_moe/compressed\_tensors\_moe\_w8a8\_int8.py", line 166, in get\_fused\_moe\_quant\_config (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return make\_int8\_moe\_quant\_config( (Worker\_TP7 pid=866) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] TypeError: install\_fl\_w8a8\_moe\_selector..make\_int8\_moe\_quant\_config\_fl() got an unexpected keyword argument 'int8\_backend' (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] WorkerProc failed to start. (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] Traceback (most recent call last): (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/v1/executor/multiproc\_executor.py", line 894, in worker\_main (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] worker = WorkerProc(\*args, \*\*kwargs) (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/v1/executor/multiproc\_executor.py", line 663, in \*\*init\*\* (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.worker.load\_model() (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/worker.py", line 443, in load\_model (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.model\_runner.load\_model(load\_dummy\_weights=load\_dummy\_weights) (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/model\_runner.py", line 5349, in load\_model (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] raise e (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm-plugin-FL/vllm\_fl/worker/model\_runner.py", line 5268, in load\_model (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] self.model = model\_loader.load\_model( (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/tracing/otel.py", line 178, in sync\_wrapper (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] return func(\*args, \*\*kwargs) (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/model\_loader/base\_loader.py", line 80, in load\_model (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] process\_weights\_after\_loading(model, model\_config, target\_device) (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] File "/workspace/vllm/vllm/model\_executor/model\_loader/utils.py", line 108, in process\_weights\_after\_loading (Worker\_TP5 pid=864) ERROR 09-28 03:10:39 [multiproc\_executor.py:927] quant\_method.process\_weights\_after\_loading(module)