DCU本身可以被识别出来:
root@r5330:/workspace/nn_train8k# python3 -c "import torch;print('pytorch version:',torch.__version__);print('DCU is',torch.cuda.is_available())"
pytorch version: 2.1.0
DCU is True
但是pytorch lighting无法正确识别DCU:
root@r5330:/workspace/nn_train8k# HYDRA_FULL_ERROR=1 bash run_8k.sh
+ set +x
run lid_8k_resnetdf34_circle_vox8_speed_2s with 2 gpus
/usr/local/lib/python3.11/site-packages/hydra/_internal/hydra.py:119: UserWarning: Future Hydra versions will no longer change working directory at job runtime by default.
See https://hydra.cc/docs/1.2/upgrades/1.1_to_1.2/changes_to_job_working_dir/ for more information.
ret = run_job(
[2025-10-28 15:08:46,611] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
{'seed': 7777, 'gpus': 2, 'find_unused_parameters': False, 'strategy': 'ddp', 'project_name': 'lid_8k', 'experiment_name': 'resnetdf34_circle', 'mod': 'default', 'stage': 'train', 'task':
'classification', 'task_mode': None, 'checkpoint': None, 'load_optim': False, 'resume': False, 'spk_offset': -1, 'trials': 'no_trials', 'tag': '${now:%Y-%m-%d_%H-%M-%S}', 'data':
{'dataloader': {'_target_': 'src.data.common.SpeakerDataLoaderConfig', 'batch_size': 32, 'num_workers': 10, 'pin_memory': True}, 'dataset': {'_target_':
'src.data.module.wav_dataset.WavDataModuleConfig', 'name': ['lang'], 'spk2wav': ['${oc.env:PWD}/data/lid_8k/spk2wav_lid8k'], 'min_dur': 4, 'sample_rate': 8000, 'speaker_num': [3],
'sample_mode': 'utt', 'collate_type': 'pad_right'}, 'pipeline': {'train_pipeline': ['augmenter'], 'augmenter': {'_target_': 'src.data.preprocess.augment.Augmenter', 'weights': [1]},
'augmentations': ['augment_add_nothing'], 'augment_add_nothing': {'_target_': 'src.data.preprocess.augment.AugmentAddNothing', 'name': 'add_nothing'}}}, 'model': {'_target_':
'src.model.resnet_circle.ResNetCircleModelConfig', 'mel_add_pre_emphasis': True, 'mel_sample_rate': 8000, 'mel_frame_length': 25, 'mel_frame_shift': 10, 'mel_num_bins': 80, 'mel_n_fft':
512, 'mel_low_cut': 0, 'mel_high_cut': 4000, 'mel_log_input': True, 'mel_cmn_window': 300, 'mel_cmn_min_window': 100, 'mel_cmn_center': False, 'mel_cmn_norm_var': False,
'resnet_feature_dim': '${model.mel_num_bins}', 'resnet_channel_num': [32, 64, 128, 256], 'resnet_layer_num': [3, 4, 6, 3], 'resnet_blocks': 'BottleneckDf', 'resnet_block_kwargs': None,
'resnet_block_drop_path': 0.1, 'resnet_pooling': 'ASP', 'resnet_embedding_dim': 512, 'resnet_dropout': 0.3, 'resnet_out_bn': True, 'circle_embedding_dim': '${model.resnet_embedding_dim}',
'circle_classes_num': '${data.dataset.speaker_num}', 'circle_margin': 0.3, 'circle_scale': 50, 'circle_subcenter_k': 1, 'circle_intertop_k': 1, 'circle_intertop_margin': 0.1,
'margin_start': 0.5, 'margin_warmup_steps': 0, 'freeze_encoder': False, 'freeze_encoder_steps': 0}, 'optim': {'algo': {'_target_': 'torch.optim.Adam', '_convert_': 'partial', 'lr': 0.001,
'weight_decay': 5e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'amsgrad': False}, 'schedule': {'scheduler': {'_target_': 'torch.optim.lr_scheduler.StepLR', 'step_size': 1, 'gamma': 1,
'last_epoch': -1, 'verbose': False}, 'monitor': None, 'interval': 'epoch', 'frequency': None, 'name': None}}, 'callback': {'lr_monitor': {'_target_':
'pytorch_lightning.callbacks.LearningRateMonitor', 'logging_interval': 'step'}, 'checkpoint': {'_target_': 'pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint', 'filename':
'{epoch}.{step}', 'monitor': 'global_step', 'mode': 'max', 'save_top_k': 10, 'every_n_train_steps': 400, 'save_last': True}}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'devices':
'${gpus}', 'accelerator': 'gpu', 'num_nodes': 1, 'min_epochs': None, 'max_epochs': 80, 'min_steps': None, 'max_steps': 48000, 'accumulate_grad_batches': 1, 'progress_bar_refresh_rate': 100,
'limit_train_batches': 1.0, 'limit_val_batches': 0, 'limit_test_batches': 0, 'fast_dev_run': False, 'precision': 16, 'sync_batchnorm': True}}
[2025-10-28 15:08:48,229][pytorch_lightning.utilities.seed][INFO] - Global seed set to 7777
/usr/local/lib/python3.11/site-packages/speechbrain/dataio/dataio.py:26: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.
torchaudio.set_audio_backend(torchaudio_backend)
/usr/local/lib/python3.11/site-packages/speechbrain/nnet/loss/stoi_loss.py:13: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.
torchaudio.set_audio_backend(torchaudio_backend)
/usr/local/lib/python3.11/site-packages/speechbrain/lobes/augment.py:28: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.
torchaudio.set_audio_backend(torchaudio_backend)
<class 'src.data.module.wav_dataset.WavDataModule'>
sample mode utt
/workspace/nn_train8k/data/lid_8k/spk2wav_lid8k: 69983 samples, 3 speakers
### lang statistics ###
training samples: 69983
training speakers: 3
<class 'src.model.resnet_circle.ResNetCircleModel'>
no_trials does not exists
Error executing job with overrides: ['project_name=lid_8k', 'experiment_name=resnetdf34_circle', 'data/dataset=swga', 'data.dataset.min_dur=2', 'data/pipeline=default_serial', 'data.dataloader.batch_size=128', 'data.dataloader.num_workers=10', 'data/dataset=lid8k', 'data.dataloader.batch_size=32', 'data.dataloader.num_workers=10', 'data/pipeline=no_aug', 'data.dataset.min_dur=4', 'model=resnet_circle', 'model.mel_sample_rate=8000', 'model.mel_high_cut=4000', 'model.resnet_block_drop_path=0.1', 'model.resnet_blocks=BottleneckDf', 'model.resnet_layer_num=[3,4,6,3]', 'model.resnet_channel_num=[32,64,128,256]', 'model.circle_margin=0.3', 'model.circle_scale=50', 'optim/algo=adam', 'optim.algo.weight_decay=5e-5', 'optim.algo.lr=0.001', 'optim/schedule=constant', 'trainer.max_steps=48000', 'callback.checkpoint.every_n_train_steps=400', 'trainer.max_epochs=80', 'gpus=2']
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 92, in _call_target
return _target_(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/env_vars_connector.py", line 38, in insert_env_defaults
return fn(self, **kwargs)
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 431, in __init__
self._accelerator_connector = AcceleratorConnector(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py", line 177, in __init__
self._validate_accelerator_type()
File "/usr/local/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py", line 248, in _validate_accelerator_type
raise ValueError(
ValueError: Mismatch between the requested accelerator type (DeviceType.GPU) and assigned device type (DeviceType.CPU).
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/nn_train8k/train.py", line 23, in <module>
train()
File "/usr/local/lib/python3.11/site-packages/hydra/main.py", line 94, in decorated_main
_run_hydra(
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
_run_app(
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 457, in _run_app
run_and_report(
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
raise ex
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
return func()
^^^^^^
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
lambda: hydra.run(
^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/hydra.py", line 132, in run
_ = ret.return_value
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/hydra/core/utils.py", line 260, in return_value
raise self._return_value
File "/usr/local/lib/python3.11/site-packages/hydra/core/utils.py", line 186, in run_job
ret.return_value = task_function(task_cfg)
^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/nn_train8k/train.py", line 20, in train
train(cfg)
File "/workspace/nn_train8k/src/train.py", line 11, in train
trainer = build_trainer(cfg)
^^^^^^^^^^^^^^^^^^
File "/workspace/nn_train8k/src/build.py", line 127, in build_trainer
trainer: Trainer = instantiate(
^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 226, in instantiate
return instantiate_node(
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 347, in instantiate_node
return _call_target(_target_, partial, args, kwargs, full_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 97, in _call_target
raise InstantiationException(msg) from e
hydra.errors.InstantiationException: Error in call to target 'pytorch_lightning.trainer.trainer.Trainer':
ValueError('Mismatch between the requested accelerator type (DeviceType.GPU) and assigned device type (DeviceType.CPU).')
full_key: trainer

暂无数据