1
7
模型应用
pytorch lighting 兼容问题
2025-10-28 15:17:37
65


DCU本身可以被识别出来:

root@r5330:/workspace/nn_train8k# python3 -c "import torch;print('pytorch version:',torch.__version__);print('DCU is',torch.cuda.is_available())"

pytorch version: 2.1.0

DCU is True


但是pytorch lighting无法正确识别DCU:


root@r5330:/workspace/nn_train8k# HYDRA_FULL_ERROR=1 bash run_8k.sh

+ set +x

run lid_8k_resnetdf34_circle_vox8_speed_2s with 2 gpus

/usr/local/lib/python3.11/site-packages/hydra/_internal/hydra.py:119: UserWarning: Future Hydra versions will no longer change working directory at job runtime by default.

See https://hydra.cc/docs/1.2/upgrades/1.1_to_1.2/changes_to_job_working_dir/ for more information.

 ret = run_job(

[2025-10-28 15:08:46,611] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)

{'seed': 7777, 'gpus': 2, 'find_unused_parameters': False, 'strategy': 'ddp', 'project_name': 'lid_8k', 'experiment_name': 'resnetdf34_circle', 'mod': 'default', 'stage': 'train', 'task':

'classification', 'task_mode': None, 'checkpoint': None, 'load_optim': False, 'resume': False, 'spk_offset': -1, 'trials': 'no_trials', 'tag': '${now:%Y-%m-%d_%H-%M-%S}', 'data':

{'dataloader': {'_target_': 'src.data.common.SpeakerDataLoaderConfig', 'batch_size': 32, 'num_workers': 10, 'pin_memory': True}, 'dataset': {'_target_':

'src.data.module.wav_dataset.WavDataModuleConfig', 'name': ['lang'], 'spk2wav': ['${oc.env:PWD}/data/lid_8k/spk2wav_lid8k'], 'min_dur': 4, 'sample_rate': 8000, 'speaker_num': [3],

'sample_mode': 'utt', 'collate_type': 'pad_right'}, 'pipeline': {'train_pipeline': ['augmenter'], 'augmenter': {'_target_': 'src.data.preprocess.augment.Augmenter', 'weights': [1]},

'augmentations': ['augment_add_nothing'], 'augment_add_nothing': {'_target_': 'src.data.preprocess.augment.AugmentAddNothing', 'name': 'add_nothing'}}}, 'model': {'_target_':

'src.model.resnet_circle.ResNetCircleModelConfig', 'mel_add_pre_emphasis': True, 'mel_sample_rate': 8000, 'mel_frame_length': 25, 'mel_frame_shift': 10, 'mel_num_bins': 80, 'mel_n_fft':

512, 'mel_low_cut': 0, 'mel_high_cut': 4000, 'mel_log_input': True, 'mel_cmn_window': 300, 'mel_cmn_min_window': 100, 'mel_cmn_center': False, 'mel_cmn_norm_var': False,

'resnet_feature_dim': '${model.mel_num_bins}', 'resnet_channel_num': [32, 64, 128, 256], 'resnet_layer_num': [3, 4, 6, 3], 'resnet_blocks': 'BottleneckDf', 'resnet_block_kwargs': None,

'resnet_block_drop_path': 0.1, 'resnet_pooling': 'ASP', 'resnet_embedding_dim': 512, 'resnet_dropout': 0.3, 'resnet_out_bn': True, 'circle_embedding_dim': '${model.resnet_embedding_dim}',

'circle_classes_num': '${data.dataset.speaker_num}', 'circle_margin': 0.3, 'circle_scale': 50, 'circle_subcenter_k': 1, 'circle_intertop_k': 1, 'circle_intertop_margin': 0.1,

'margin_start': 0.5, 'margin_warmup_steps': 0, 'freeze_encoder': False, 'freeze_encoder_steps': 0}, 'optim': {'algo': {'_target_': 'torch.optim.Adam', '_convert_': 'partial', 'lr': 0.001,

'weight_decay': 5e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'amsgrad': False}, 'schedule': {'scheduler': {'_target_': 'torch.optim.lr_scheduler.StepLR', 'step_size': 1, 'gamma': 1,

'last_epoch': -1, 'verbose': False}, 'monitor': None, 'interval': 'epoch', 'frequency': None, 'name': None}}, 'callback': {'lr_monitor': {'_target_':

'pytorch_lightning.callbacks.LearningRateMonitor', 'logging_interval': 'step'}, 'checkpoint': {'_target_': 'pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint', 'filename':

'{epoch}.{step}', 'monitor': 'global_step', 'mode': 'max', 'save_top_k': 10, 'every_n_train_steps': 400, 'save_last': True}}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'devices':

'${gpus}', 'accelerator': 'gpu', 'num_nodes': 1, 'min_epochs': None, 'max_epochs': 80, 'min_steps': None, 'max_steps': 48000, 'accumulate_grad_batches': 1, 'progress_bar_refresh_rate': 100,

'limit_train_batches': 1.0, 'limit_val_batches': 0, 'limit_test_batches': 0, 'fast_dev_run': False, 'precision': 16, 'sync_batchnorm': True}}

[2025-10-28 15:08:48,229][pytorch_lightning.utilities.seed][INFO] - Global seed set to 7777

/usr/local/lib/python3.11/site-packages/speechbrain/dataio/dataio.py:26: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.

 torchaudio.set_audio_backend(torchaudio_backend)

/usr/local/lib/python3.11/site-packages/speechbrain/nnet/loss/stoi_loss.py:13: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.

 torchaudio.set_audio_backend(torchaudio_backend)

/usr/local/lib/python3.11/site-packages/speechbrain/lobes/augment.py:28: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.

 torchaudio.set_audio_backend(torchaudio_backend)

<class 'src.data.module.wav_dataset.WavDataModule'>

sample mode utt

/workspace/nn_train8k/data/lid_8k/spk2wav_lid8k: 69983 samples, 3 speakers

### lang statistics ###

training samples: 69983

training speakers: 3

<class 'src.model.resnet_circle.ResNetCircleModel'>

no_trials does not exists

Error executing job with overrides: ['project_name=lid_8k', 'experiment_name=resnetdf34_circle', 'data/dataset=swga', 'data.dataset.min_dur=2', 'data/pipeline=default_serial', 'data.dataloader.batch_size=128', 'data.dataloader.num_workers=10', 'data/dataset=lid8k', 'data.dataloader.batch_size=32', 'data.dataloader.num_workers=10', 'data/pipeline=no_aug', 'data.dataset.min_dur=4', 'model=resnet_circle', 'model.mel_sample_rate=8000', 'model.mel_high_cut=4000', 'model.resnet_block_drop_path=0.1', 'model.resnet_blocks=BottleneckDf', 'model.resnet_layer_num=[3,4,6,3]', 'model.resnet_channel_num=[32,64,128,256]', 'model.circle_margin=0.3', 'model.circle_scale=50', 'optim/algo=adam', 'optim.algo.weight_decay=5e-5', 'optim.algo.lr=0.001', 'optim/schedule=constant', 'trainer.max_steps=48000', 'callback.checkpoint.every_n_train_steps=400', 'trainer.max_epochs=80', 'gpus=2']

Traceback (most recent call last):

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 92, in _call_target

   return _target_(*args, **kwargs)

          ^^^^^^^^^^^^^^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/env_vars_connector.py", line 38, in insert_env_defaults

   return fn(self, **kwargs)

          ^^^^^^^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 431, in __init__

   self._accelerator_connector = AcceleratorConnector(

                                 ^^^^^^^^^^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py", line 177, in __init__

   self._validate_accelerator_type()

 File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py", line 248, in _validate_accelerator_type

   raise ValueError(

ValueError: Mismatch between the requested accelerator type (DeviceType.GPU) and assigned device type (DeviceType.CPU).


The above exception was the direct cause of the following exception:


Traceback (most recent call last):

 File "/workspace/nn_train8k/train.py", line 23, in <module>

   train()

 File "/usr/local/lib/python3.11/site-packages/hydra/main.py", line 94, in decorated_main

   _run_hydra(

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra

   _run_app(

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 457, in _run_app

   run_and_report(

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 223, in run_and_report

   raise ex

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 220, in run_and_report

   return func()

          ^^^^^^

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 458, in <lambda>

   lambda: hydra.run(

           ^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/hydra.py", line 132, in run

   _ = ret.return_value

       ^^^^^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/hydra/core/utils.py", line 260, in return_value

   raise self._return_value

 File "/usr/local/lib/python3.11/site-packages/hydra/core/utils.py", line 186, in run_job

   ret.return_value = task_function(task_cfg)

                      ^^^^^^^^^^^^^^^^^^^^^^^

 File "/workspace/nn_train8k/train.py", line 20, in train

   train(cfg)

 File "/workspace/nn_train8k/src/train.py", line 11, in train

   trainer = build_trainer(cfg)

             ^^^^^^^^^^^^^^^^^^

 File "/workspace/nn_train8k/src/build.py", line 127, in build_trainer

   trainer: Trainer = instantiate(

                      ^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 226, in instantiate

   return instantiate_node(

          ^^^^^^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 347, in instantiate_node

   return _call_target(_target_, partial, args, kwargs, full_key)

          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 97, in _call_target

   raise InstantiationException(msg) from e

hydra.errors.InstantiationException: Error in call to target 'pytorch_lightning.trainer.trainer.Trainer':

ValueError('Mismatch between the requested accelerator type (DeviceType.GPU) and assigned device type (DeviceType.CPU).')

full_key: trainer

全部评论(2)
只看楼主
F
future
楼主
2楼
DCU:K100_AI
2025/10/28
F
future
楼主
1楼
请问如何解决这个问题呢
2025/10/28
没有更多内容了
--
楼主
Level.2
0 帖子
楼主热帖
暂无数据