Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unified all mpi assert mechanisms #2465

Merged
merged 8 commits into from
Oct 4, 2023
Merged

Unified all mpi assert mechanisms #2465

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
88f0702
Unified all mpi assert mechanisms
Sep 28, 2023
fa06fcc
Merge branch 'master' into cornu/clean_mpispike
Sep 28, 2023
427ad30
Merge branch 'master' into cornu/clean_mpispike
Sep 28, 2023
6c48766
Merge branch 'master' into cornu/clean_mpispike
alexsavulescu Sep 28, 2023
7ad9347
Create an helper function to get the MPI_Op from type (#2560)
Sep 29, 2023
7a41b6e
Merge branch 'master' into cornu/clean_mpispike
Oct 2, 2023
8686690
Merge branch 'master' into cornu/clean_mpispike
Oct 3, 2023
ae02186
Reduce sonarcloud warnings
Oct 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 11 additions & 26 deletions src/coreneuron/mpi/lib/mpispike.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,46 +289,31 @@ void nrnmpi_barrier_impl() {
MPI_Barrier(nrnmpi_comm);
}

double nrnmpi_dbl_allreduce_impl(double x, int type) {
double result;
static MPI_Op type2OP(int type) {
MPI_Op tt;
if (type == 1) {
tt = MPI_SUM;
return MPI_SUM;
} else if (type == 2) {
tt = MPI_MAX;
return MPI_MAX;
} else {
tt = MPI_MIN;
return MPI_MIN;
}
MPI_Allreduce(&x, &result, 1, MPI_DOUBLE, tt, nrnmpi_comm);
}

double nrnmpi_dbl_allreduce_impl(double x, int type) {
double result;
MPI_Allreduce(&x, &result, 1, MPI_DOUBLE, type2OP(type), nrnmpi_comm);
return result;
}

void nrnmpi_dbl_allreduce_vec_impl(double* src, double* dest, int cnt, int type) {
MPI_Op tt;
assert(src != dest);
if (type == 1) {
tt = MPI_SUM;
} else if (type == 2) {
tt = MPI_MAX;
} else {
tt = MPI_MIN;
}
MPI_Allreduce(src, dest, cnt, MPI_DOUBLE, tt, nrnmpi_comm);
return;
MPI_Allreduce(src, dest, cnt, MPI_DOUBLE, type2OP(type), nrnmpi_comm);
}

void nrnmpi_long_allreduce_vec_impl(long* src, long* dest, int cnt, int type) {
MPI_Op tt;
assert(src != dest);
if (type == 1) {
tt = MPI_SUM;
} else if (type == 2) {
tt = MPI_MAX;
} else {
tt = MPI_MIN;
}
MPI_Allreduce(src, dest, cnt, MPI_LONG, tt, nrnmpi_comm);
return;
MPI_Allreduce(src, dest, cnt, MPI_LONG, type2OP(type), nrnmpi_comm);
}

#if NRN_MULTISEND
Expand Down
57 changes: 24 additions & 33 deletions src/nrnmpi/bbsmpipack.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,7 @@
#include <nrnmpi_impl.h>
#include <hocdec.h>

#if 0
#define guard(f) nrn_assert(f == MPI_SUCCESS)
#else
#define guard(f) \
pramodk marked this conversation as resolved.
Show resolved Hide resolved
{ \
int _i = f; \
if (_i != MPI_SUCCESS) { \
printf("%s %d\n", #f, _i); \
assert(0); \
} \
}
#endif
#define nrn_mpi_assert(arg) nrn_assert(arg == MPI_SUCCESS)

#define nrnmpidebugleak 0
#define debug 0
Expand Down Expand Up @@ -69,7 +58,7 @@ static void unpack(void* buf, int count, int my_datatype, bbsmpibuf* r, const ch
r->size);
#endif
assert(r->upkpos >= 0 && r->size >= r->upkpos);
guard(MPI_Unpack(r->buf, r->size, &r->upkpos, type, 2, MPI_INT, nrn_bbs_comm));
nrn_mpi_assert(MPI_Unpack(r->buf, r->size, &r->upkpos, type, 2, MPI_INT, nrn_bbs_comm));
#if debug
printf("%d unpack r=%p size=%d upkpos=%d type[0]=%d datatype=%d type[1]=%d count=%d\n",
nrnmpi_myid_bbs,
Expand All @@ -93,7 +82,8 @@ static void unpack(void* buf, int count, int my_datatype, bbsmpibuf* r, const ch
}
assert(type[0] == my_datatype);
assert(type[1] == count);
guard(MPI_Unpack(r->buf, r->size, &r->upkpos, buf, count, mytypes[my_datatype], nrn_bbs_comm));
nrn_mpi_assert(
MPI_Unpack(r->buf, r->size, &r->upkpos, buf, count, mytypes[my_datatype], nrn_bbs_comm));
}

void nrnmpi_upkbegin(bbsmpibuf* r) {
Expand All @@ -111,12 +101,12 @@ void nrnmpi_upkbegin(bbsmpibuf* r) {
hoc_execerror("subworld process with nhost > 0 cannot use", "the bulletin board");
}
r->upkpos = 0;
guard(MPI_Unpack(r->buf, r->size, &r->upkpos, &p, 1, MPI_INT, nrn_bbs_comm));
nrn_mpi_assert(MPI_Unpack(r->buf, r->size, &r->upkpos, &p, 1, MPI_INT, nrn_bbs_comm));
if (p > r->size) {
printf("\n %d nrnmpi_upkbegin keypos=%d size=%d\n", nrnmpi_myid_bbs, p, r->size);
}
assert(p <= r->size);
guard(MPI_Unpack(r->buf, r->size, &p, &type, 1, MPI_INT, nrn_bbs_comm));
nrn_mpi_assert(MPI_Unpack(r->buf, r->size, &p, &type, 1, MPI_INT, nrn_bbs_comm));
#if debug
printf("%d nrnmpi_upkbegin type=%d keypos=%d\n", nrnmpi_myid_bbs, type, p);
#endif
Expand Down Expand Up @@ -230,7 +220,7 @@ void nrnmpi_pkbegin(bbsmpibuf* r) {
printf(
"%d nrnmpi_pkbegin %p size=%d pkposition=%d\n", nrnmpi_myid_bbs, r, r->size, r->pkposition);
#endif
guard(MPI_Pack(&type, 1, MPI_INT, r->buf, r->size, &r->pkposition, nrn_bbs_comm));
nrn_mpi_assert(MPI_Pack(&type, 1, MPI_INT, r->buf, r->size, &r->pkposition, nrn_bbs_comm));
}

void nrnmpi_enddata(bbsmpibuf* r) {
Expand All @@ -240,23 +230,23 @@ void nrnmpi_enddata(bbsmpibuf* r) {
#if debug
printf("%d nrnmpi_enddata %p size=%d pkposition=%d\n", nrnmpi_myid_bbs, r, r->size, p);
#endif
guard(MPI_Pack_size(1, MPI_INT, nrn_bbs_comm, &isize));
nrn_mpi_assert(MPI_Pack_size(1, MPI_INT, nrn_bbs_comm, &isize));
oldsize = r->size;
resize(r, r->pkposition + isize);
#if debug
if (oldsize < r->pkposition + isize) {
printf("%d %p need %d more. end up with total of %d\n", nrnmpi_myid_bbs, r, isize, r->size);
}
#endif
guard(MPI_Pack(&type, 1, MPI_INT, r->buf, r->size, &r->pkposition, nrn_bbs_comm));
nrn_mpi_assert(MPI_Pack(&type, 1, MPI_INT, r->buf, r->size, &r->pkposition, nrn_bbs_comm));
#if debug
printf("%d nrnmpi_enddata buf=%p size=%d pkposition=%d\n",
nrnmpi_myid_bbs,
r->buf,
r->size,
r->pkposition);
#endif
guard(MPI_Pack(&p, 1, MPI_INT, r->buf, r->size, &type, nrn_bbs_comm));
nrn_mpi_assert(MPI_Pack(&p, 1, MPI_INT, r->buf, r->size, &type, nrn_bbs_comm));
#if debug
printf("%d after nrnmpi_enddata, %d was packed at beginning and 0 was packed before %d\n",
nrnmpi_myid_bbs,
Expand All @@ -278,8 +268,8 @@ static void pack(void* inbuf, int incount, int my_datatype, bbsmpibuf* r, const
r->pkposition,
e);
#endif
guard(MPI_Pack_size(incount, mytypes[my_datatype], nrn_bbs_comm, &dsize));
guard(MPI_Pack_size(2, MPI_INT, nrn_bbs_comm, &isize));
nrn_mpi_assert(MPI_Pack_size(incount, mytypes[my_datatype], nrn_bbs_comm, &dsize));
nrn_mpi_assert(MPI_Pack_size(2, MPI_INT, nrn_bbs_comm, &isize));
oldsize = r->size;
resize(r, r->pkposition + dsize + isize);
#if debug
Expand All @@ -293,8 +283,8 @@ static void pack(void* inbuf, int incount, int my_datatype, bbsmpibuf* r, const
#endif
type[0] = my_datatype;
type[1] = incount;
guard(MPI_Pack(type, 2, MPI_INT, r->buf, r->size, &r->pkposition, nrn_bbs_comm));
guard(MPI_Pack(
nrn_mpi_assert(MPI_Pack(type, 2, MPI_INT, r->buf, r->size, &r->pkposition, nrn_bbs_comm));
nrn_mpi_assert(MPI_Pack(
inbuf, incount, mytypes[my_datatype], r->buf, r->size, &r->pkposition, nrn_bbs_comm));
#if debug
printf("%d pack done pkposition=%d\n", nrnmpi_myid_bbs, r->pkposition);
Expand Down Expand Up @@ -357,9 +347,9 @@ void nrnmpi_bbssend(int dest, int tag, bbsmpibuf* r) {

if (r) {
assert(r->buf && r->keypos <= r->size);
guard(MPI_Send(r->buf, r->size, MPI_PACKED, dest, tag, nrn_bbs_comm));
nrn_mpi_assert(MPI_Send(r->buf, r->size, MPI_PACKED, dest, tag, nrn_bbs_comm));
} else {
guard(MPI_Send(NULL, 0, MPI_PACKED, dest, tag, nrn_bbs_comm));
nrn_mpi_assert(MPI_Send(NULL, 0, MPI_PACKED, dest, tag, nrn_bbs_comm));
}
errno = 0;
#if debug
Expand All @@ -376,8 +366,8 @@ int nrnmpi_bbsrecv(int source, bbsmpibuf* r) {
#if debug
printf("%d nrnmpi_bbsrecv %p\n", nrnmpi_myid_bbs, r);
#endif
guard(MPI_Probe(source, MPI_ANY_TAG, nrn_bbs_comm, &status));
guard(MPI_Get_count(&status, MPI_PACKED, &size));
nrn_mpi_assert(MPI_Probe(source, MPI_ANY_TAG, nrn_bbs_comm, &status));
nrn_mpi_assert(MPI_Get_count(&status, MPI_PACKED, &size));
#if debug
printf("%d nrnmpi_bbsrecv probe size=%d source=%d tag=%d\n",
nrnmpi_myid_bbs,
Expand All @@ -386,7 +376,8 @@ int nrnmpi_bbsrecv(int source, bbsmpibuf* r) {
status.MPI_TAG);
#endif
resize(r, size);
guard(MPI_Recv(r->buf, r->size, MPI_PACKED, source, MPI_ANY_TAG, nrn_bbs_comm, &status));
nrn_mpi_assert(
MPI_Recv(r->buf, r->size, MPI_PACKED, source, MPI_ANY_TAG, nrn_bbs_comm, &status));
errno = 0;
/* Some MPI implementations limit tags to be less than full MPI_INT domain
In the past we allowed TODO mesages to have tags > 20 (FIRSTID of src/parallel/bbssrv.h)
Expand Down Expand Up @@ -425,28 +416,28 @@ int nrnmpi_bbssendrecv(int dest, int tag, bbsmpibuf* s, bbsmpibuf* r) {
int nrnmpi_iprobe(int* size, int* tag, int* source) {
int flag = 0;
MPI_Status status;
guard(MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, nrn_bbs_comm, &flag, &status));
nrn_mpi_assert(MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, nrn_bbs_comm, &flag, &status));
if (flag) {
if (source)
*source = status.MPI_SOURCE;
if (tag)
*tag = status.MPI_TAG;
if (size)
guard(MPI_Get_count(&status, MPI_PACKED, size));
nrn_mpi_assert(MPI_Get_count(&status, MPI_PACKED, size));
}
return flag;
}

void nrnmpi_probe(int* size, int* tag, int* source) {
int flag = 0;
MPI_Status status;
guard(MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, nrn_bbs_comm, &status));
nrn_mpi_assert(MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, nrn_bbs_comm, &status));
if (source)
*source = status.MPI_SOURCE;
if (tag)
*tag = status.MPI_TAG;
if (size)
guard(MPI_Get_count(&status, MPI_PACKED, size));
nrn_mpi_assert(MPI_Get_count(&status, MPI_PACKED, size));
}

bbsmpibuf* nrnmpi_newbuf(int size) {
Expand Down
Loading