光合开发者社区-论坛

#include <stdio.h> #include <hip/hip_runtime.h> #include <mpi.h> #include <iostream> #define NR_END 1 #define NX 6 #define NY 16 #define NZ 16 using namespace std; double ***cube(int xl, int xr, int yl, int yr, int zl, int zr) { int i, j, nrow = xr - xl + 1, ncol = yr - yl + 1, ndep = zr - zl + 1; double ***t; t = (double ***)malloc(((nrow + NR_END) * sizeof(double **))); t += NR_END; t -= xl; t[xl] = (double **)malloc(((nrow * ncol + NR_END) * sizeof(double *))); t[xl] += NR_END; t[xl] -= yl; t[xl][yl] = (double *)malloc(((nrow * ncol * ndep + NR_END) * sizeof(double))); t[xl][yl] += NR_END; t[xl][yl] -= zl; for (j = yl + 1; j <= yr; j++) t[xl][j] = t[xl][j - 1] + ndep; for (i = xl + 1; i <= xr; i++) { t[i] = t[i - 1] + ncol; t[i][yl] = t[i - 1][yl] + ncol * ndep; for (j = yl + 1; j <= yr; j++) t[i][j] = t[i][j - 1] + ndep; } return t; } int main(int argc, char **argv) { int myrank, nprocs; int ndcus; char prcname[100]; MPI_Status status; MPI_Init(NULL, NULL); MPI_Comm_rank(MPI_COMM_WORLD, &myrank); MPI_Get_processor_name(prcname, &nprocs); hipGetDeviceCount(&ndcus); printf("node:%s\tmyrank:%d\tndcus:%d\n", prcname, myrank, ndcus); int nbytes = sizeof(double) * (NX + 2) * (NY + 2) * (NZ + 2); double ***p, *d_p; p = cube(0, NX + 1, 0, NY + 1, 0, NZ + 1); for (int i = 0; i <= NX + 1; ++i) for (int j = 0; j <= NY + 1; ++j) for (int k = 0; k <= NZ + 1; ++k) { p[i][j][k] = myrank + 1; } hipSetDevice(myrank); hipMalloc(&d_p, nbytes); hipMemcpy(d_p, &p[0][0][0], nbytes, hipMemcpyHostToDevice); if (myrank == 0) { MPI_Send(d_p, 2047, MPI_DOUBLE, 1, myrank, MPI_COMM_WORLD); } if (myrank == 1) { MPI_Recv(d_p, 2047, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, &status); } hipMemcpy(&p[0][0][0], d_p, nbytes, hipMemcpyDeviceToHost); MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return 0; }

cma_ep.c:84 process_vm_readv(pid=431 {0x2aefdf800000,16384}-->{0x2b4817c00000,16384}) returned -1: Bad address ==== backtrace (tid: 432) ==== 0 0x00000000000027c6 uct_cma_ep_tx_error() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/uct/sm/scopy/cma/cma_ep.c:81 1 0x0000000000002ad8 uct_cma_ep_tx() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/uct/sm/scopy/cma/cma_ep.c:114 2 0x000000000001bb65 uct_scopy_ep_progress_tx() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/uct/sm/scopy/base/scopy_ep.c:151 3 0x00000000000520c0 ucs_arbiter_dispatch_nonempty() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/ucs/datastruct/arbiter.c:321 4 0x000000000001b6b9 ucs_arbiter_dispatch() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/ucs/datastruct/arbiter.h:386 5 0x000000000005273c ucs_callbackq_slow_proxy() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/ucs/datastruct/callbackq.c:404 6 0x00000000000415ea ucs_callbackq_dispatch() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/ucs/datastruct/callbackq.h:211 7 0x00000000000415ea uct_worker_progress() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/uct/api/uct.h:2638 8 0x00000000000415ea ucp_worker_progress() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/ucx-1.13.0/src/ucp/core/ucp_worker.c:2782 9 0x00000000000042a7 mca_pml_ucx_recv() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/openmpi-gitclone/ompi/mca/pml/ucx/pml_ucx.c:646 10 0x000000000007b6cd PMPI_Recv() /opt/hpc/software/mpi/hpcx/hpcx-v2.11-gcc-MLNX_OFED_LINUX-5-redhat7-cuda11-gdrcopy2-nccl2.11-x86_64/sources/openmpi-gitclone/ompi/mpi/c/profile/precv.c:82 11 0x0000000000401cf6 main() ???:0 12 0x00000000000223d5 __libc_start_main() ???:0 13 0x0000000000400cb9 _start() ???:0 ================================= [c13r1n02:00432] *** Process received signal *** [c13r1n02:00432] Signal: Aborted (6) [c13r1n02:00432] Signal code: (-6) [c13r1n02:00432] [ 0] /lib64/libpthread.so.0(+0xf5d0)[0x2aeb89b195d0] [c13r1n02:00432] [ 1] /lib64/libc.so.6(gsignal+0x37)[0x2aeb92472207] [c13r1n02:00432] [ 2] /lib64/libc.so.6(abort+0x148)[0x2aeb924738f8] [c13r1n02:00432] [ 3] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libucs.so.0(ucs_fatal_error_message+0x55)[0x2aeb9df9d5d5] [c13r1n02:00432] [ 4] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libucs.so.0(ucs_log_default_handler+0xf27)[0x2aeb9dfa28e7] [c13r1n02:00432] [ 5] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libucs.so.0(ucs_log_dispatch+0xe1)[0x2aeb9dfa2c81] [c13r1n02:00432] [ 6] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/ucx/libuct_cma.so.0(+0x27c6)[0x2aeb9f48c7c6] [c13r1n02:00432] [ 7] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/ucx/libuct_cma.so.0(uct_cma_ep_tx+0x158)[0x2aeb9f48cad8] [c13r1n02:00432] [ 8] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libuct.so.0(uct_scopy_ep_progress_tx+0x55)[0x2aeb9dd23b65] [c13r1n02:00432] [ 9] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libucs.so.0(ucs_arbiter_dispatch_nonempty+0xa0)[0x2aeb9df940c0] [c13r1n02:00432] [10] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libuct.so.0(uct_scopy_iface_progress+0x59)[0x2aeb9dd236b9] [c13r1n02:00432] [11] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libucs.so.0(+0x5273c)[0x2aeb9df9473c] [c13r1n02:00432] [12] /public/software/mpi/hpcx/v2.11.0/ucx_without_rocm/lib/libucp.so.0(ucp_worker_progress+0x6a)[0x2aeb9da8d5ea] [c13r1n02:00432] [13] /public/software/mpi/hpcx/v2.11.0/gcc-7.3.1/lib/openmpi/mca_pml_ucx.so(mca_pml_ucx_recv+0x127)[0x2aeb9d63f2a7] [c13r1n02:00432] [14] /public/software/mpi/hpcx/v2.11.0/gcc-7.3.1/lib/libmpi.so.40(MPI_Recv+0x2d)[0x2aeb91e806cd] [c13r1n02:00432] [15] ./test[0x401cf6] [c13r1n02:00432] [16] /lib64/libc.so.6(__libc_start_main+0xf5)[0x2aeb9245e3d5] [c13r1n02:00432] [17] ./test[0x400cb9] [c13r1n02:00432] *** End of error message *** -------------------------------------------------------------------------- Primary job terminated normally, but 1 process returned a non-zero exit code. Per user-direction, the job has been aborted. -------------------------------------------------------------------------- -------------------------------------------------------------------------- mpirun noticed that process rank 1 with PID 0 on node c13r1n02 exited on signal 6 (Aborted).

查阅文档发现需要加上-mca pml ucx选项，但是这只在跨节点的时候有效。现在假如有2个节点，每个节点4个进程4个DCU，同一个节点下的DCU之间通信时，使用该选项还是会出现问题。