#SBATCH -N 2
#SBATCH -n 8
#SBATCH -p normal
#SBATCH --gres=dcu:8

以上方这种方式提交任务，报错：Batch job submission failed: Requested node configuration is not available

In run.slurm
#!/bin/bash
#SBATCH -J cuda
#SBATCH -N 2
#SBATCH -n 8
#SBATCH -p normal
#SBATCH --gres=dcu:4
#SBATCH -o logs/job.%j.log

source /public/software/apps/anaconda3/5.2.0/etc/profile.d/conda.sh
conda activate /public/software/apps/anaconda3/5.2.0/envs/pytorch1.10.0a0-py37-dtk22.04.2

export NCCL_P2P_DISABLE=1
module purge
module load compiler/devtoolset/7.3.1
module load mpi/hpcx/2.7.4/gcc-7.3.1

python -u ~/alexdir/deepspeed/src/cuda.py

以上方方式申请两个节点时，只能用到四张dcu（理论应该是2*4=8张卡）

cuda .py and output is in picture