{"id":1406,"date":"2015-05-18T15:19:33","date_gmt":"2015-05-18T20:19:33","guid":{"rendered":"http:\/\/jungwon.kim\/blog\/?p=1406"},"modified":"2015-05-18T15:19:33","modified_gmt":"2015-05-18T20:19:33","slug":"gpudirect-cuda-aware-mpi","status":"publish","type":"post","link":"https:\/\/blog.jungwon.kim\/?p=1406","title":{"rendered":"GPUDirect: CUDA aware MPI"},"content":{"rendered":"

http:\/\/keeneland.gatech.edu\/software\/gpudirect<\/a><\/p>\n

https:\/\/www.olcf.ornl.gov\/tutorials\/gpudirect-mpich-enabled-cuda\/<\/a><\/p>\n

CUDA C<\/h3>\n

direct.cpp<\/h4>\n
\n
\n\n\n\n
\n
\n
#include <stdio.h><\/code><\/div>\n
#include <stdlib.h><\/code><\/div>\n
#include <cuda_runtime.h><\/code><\/div>\n
#include <mpi.h><\/code><\/div>\n
<\/div>\n
int<\/code> main( <\/code>int<\/code> argc, <\/code>char<\/code>** argv )<\/code><\/div>\n
{<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>MPI_Init (&argc, &argv);<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>int<\/code> direct;<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>int<\/code> rank, size;<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>int<\/code> *h_buff = NULL;<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>int<\/code> *d_rank = NULL;<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>int<\/code> *d_buff = NULL;<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>size_t<\/code> bytes;<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>int<\/code> i;<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>\/\/ Ensure that RDMA ENABLED CUDA is set correctly<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>direct = <\/code>getenv<\/code>(<\/code>\"MPICH_RDMA_ENABLED_CUDA\"<\/code>)==NULL?0:<\/code>atoi<\/code>(<\/code>getenv<\/code> (<\/code>\"MPICH_RDMA_ENABLED_CUDA\"<\/code>));<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>if<\/code>(direct != 1){<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>printf<\/code> (<\/code>\"MPICH_RDMA_ENABLED_CUDA not enabled!n\"<\/code>);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>exit<\/code> (EXIT_FAILURE);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>}<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>\/\/ Get MPI rank and size<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>MPI_Comm_rank (MPI_COMM_WORLD, &rank);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>MPI_Comm_size (MPI_COMM_WORLD, &size);<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>\/\/ Allocate host and device buffers and copy rank value to GPU<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>bytes = size*<\/code>sizeof<\/code>(<\/code>int<\/code>);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>h_buff = (<\/code>int<\/code>*)<\/code>malloc<\/code>(bytes);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>cudaMalloc(&d_buff, bytes);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>cudaMalloc(&d_rank, <\/code>sizeof<\/code>(<\/code>int<\/code>));<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>cudaMemcpy(d_rank, &rank, <\/code>sizeof<\/code>(<\/code>int<\/code>), cudaMemcpyHostToDevice);<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>\/\/ Preform Allgather using device buffer<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>MPI_Allgather(d_rank, 1, MPI_INT, d_buff, 1, MPI_INT, MPI_COMM_WORLD);<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>\/\/ Check that the GPU buffer is correct<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>cudaMemcpy(h_buff, d_buff, bytes, cudaMemcpyDeviceToHost);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>for<\/code>(i=0; i<size; i++){<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>if<\/code>(h_buff[i] != i) {<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>printf<\/code> (<\/code>\"Alltoall Failed!n\"<\/code>);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>exit<\/code> (EXIT_FAILURE);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>}<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>}<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>if<\/code>(rank==0)<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>printf<\/code>(<\/code>\"Success!n\"<\/code>);<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>\/\/ Clean up<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>free<\/code>(h_buff);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>cudaFree(d_buff);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>cudaFree(d_rank);<\/code><\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>MPI_Finalize();<\/code><\/div>\n
<\/div>\n
\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0<\/code>return<\/code> 0;<\/code><\/div>\n
}<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n

Compiling<\/h4>\n

For ease of compiling the GNU environment will be used.<\/p>\n

\n
\n\n\n\n
\n
\n
$ module load cudatoolkit<\/code><\/div>\n
$ module switch PrgEnv-pgi PrgEnv-gnu<\/code><\/div>\n
$ CC -lcudart direct.cpp -o direct.out<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n

Running<\/h4>\n
\n
\n\n\n\n
\n
\n
$ export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH<\/code><\/div>\n
$ export MPICH_RDMA_ENABLED_CUDA=1<\/code><\/div>\n
$ aprun -n2 -N1 .\/direct.out<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"

http:\/\/keeneland.gatech.edu\/software\/gpudirect https:\/\/www.olcf.ornl.gov\/tutorials\/gpudirect-mpich-enabled-cuda\/ CUDA C direct.cpp #include <stdio.h> #include <stdlib.h> #include <cuda_runtime.h> #include <mpi.h> int main( int argc, char** argv ) { \u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0MPI_Init (&argc, &argv); \u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0int direct; \u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0int rank, size; \u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0int *h_buff = NULL; \u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0int *d_rank = NULL; \u00c2\u00a0\u00c2\u00a0\u00c2\u00a0\u00c2\u00a0int *d_buff =<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[4],"tags":[],"class_list":["post-1406","post","type-post","status-publish","format-standard","hentry","category-cs"],"_links":{"self":[{"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=\/wp\/v2\/posts\/1406","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1406"}],"version-history":[{"count":0,"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=\/wp\/v2\/posts\/1406\/revisions"}],"wp:attachment":[{"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1406"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1406"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.jungwon.kim\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1406"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}