Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix segfaults for MPCD with large particle counts under MPI #1897

Merged
merged 12 commits into from
Nov 27, 2024
Merged

Fix segfaults for MPCD with large particle counts under MPI #1897

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f13ca19
Raise exceptions in serialized MPI methods
mphoward Sep 26, 2024
678120c
Create custom MPI datatypes for HOOMD data
mphoward Sep 26, 2024
883adca
Scatter snapshots using custom datatype
mphoward Sep 26, 2024
cfc1058
Remove other uses of serialized bcast from MPCD code
mphoward Sep 26, 2024
f23d6e0
Don't use thrust
mphoward Sep 26, 2024
b116410
Use new data structure to gather snapshots
mphoward Nov 20, 2024
379afa9
Free created types in destructors
mphoward Nov 20, 2024
558e998
Use derived types in base communicator
mphoward Nov 20, 2024
604ba15
Fix compile error in non-MPI builds
mphoward Nov 20, 2024
6e1b5de
Merge branch 'trunk-major' into fix-mpcd-large-snapshots
mphoward Nov 27, 2024
43e36b2
Merge branch 'trunk-major' into fix-mpcd-large-snapshots
joaander Nov 27, 2024
a389953
Mention MPCD initialization fix.
joaander Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 40 additions & 41 deletions hoomd/Communicator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1297,47 +1297,46 @@ Communicator::Communicator(std::shared_ptr<SystemDefinition> sysdef,

initializeNeighborArrays();

/* create a type for pdata_element */
const int nitems = 14;
int blocklengths[14] = {4, 4, 3, 1, 1, 3, 1, 4, 4, 3, 1, 4, 4, 6};
MPI_Datatype types[14] = {MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_INT,
MPI_UNSIGNED,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_UNSIGNED,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR};
MPI_Aint offsets[14];

offsets[0] = offsetof(detail::pdata_element, pos);
offsets[1] = offsetof(detail::pdata_element, vel);
offsets[2] = offsetof(detail::pdata_element, accel);
offsets[3] = offsetof(detail::pdata_element, charge);
offsets[4] = offsetof(detail::pdata_element, diameter);
offsets[5] = offsetof(detail::pdata_element, image);
offsets[6] = offsetof(detail::pdata_element, body);
offsets[7] = offsetof(detail::pdata_element, orientation);
offsets[8] = offsetof(detail::pdata_element, angmom);
offsets[9] = offsetof(detail::pdata_element, inertia);
offsets[10] = offsetof(detail::pdata_element, tag);
offsets[11] = offsetof(detail::pdata_element, net_force);
offsets[12] = offsetof(detail::pdata_element, net_torque);
offsets[13] = offsetof(detail::pdata_element, net_virial);

MPI_Datatype tmp;
MPI_Type_create_struct(nitems, blocklengths, offsets, types, &tmp);
MPI_Type_commit(&tmp);

MPI_Type_create_resized(tmp, 0, sizeof(detail::pdata_element), &m_mpi_pdata_element);
MPI_Type_commit(&m_mpi_pdata_element);
MPI_Type_free(&tmp);
// create a type for pdata_element
{
const MPI_Datatype mpi_scalar3 = m_exec_conf->getMPIConfig()->getScalar3Datatype();
const MPI_Datatype mpi_scalar4 = m_exec_conf->getMPIConfig()->getScalar4Datatype();
const unsigned int nitems = 14;
int blocklengths[nitems] = {1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 6};
MPI_Datatype types[nitems] = {mpi_scalar4,
mpi_scalar4,
mpi_scalar3,
MPI_HOOMD_SCALAR,
MPI_HOOMD_SCALAR,
MPI_INT,
MPI_UNSIGNED,
mpi_scalar4,
mpi_scalar4,
mpi_scalar3,
MPI_UNSIGNED,
mpi_scalar4,
mpi_scalar4,
MPI_HOOMD_SCALAR};
MPI_Aint offsets[nitems] = {offsetof(detail::pdata_element, pos),
offsetof(detail::pdata_element, vel),
offsetof(detail::pdata_element, accel),
offsetof(detail::pdata_element, charge),
offsetof(detail::pdata_element, diameter),
offsetof(detail::pdata_element, image),
offsetof(detail::pdata_element, body),
offsetof(detail::pdata_element, orientation),
offsetof(detail::pdata_element, angmom),
offsetof(detail::pdata_element, inertia),
offsetof(detail::pdata_element, tag),
offsetof(detail::pdata_element, net_force),
offsetof(detail::pdata_element, net_torque),
offsetof(detail::pdata_element, net_virial)};

MPI_Datatype tmp;
MPI_Type_create_struct(nitems, blocklengths, offsets, types, &tmp);
MPI_Type_create_resized(tmp, 0, sizeof(detail::pdata_element), &m_mpi_pdata_element);
MPI_Type_commit(&m_mpi_pdata_element);
}
}

//! Destructor
Expand Down
40 changes: 30 additions & 10 deletions hoomd/HOOMDMPI.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ template<typename T> void bcast(T& val, unsigned int root, const MPI_Comm mpi_co
MPI_Comm_rank(mpi_comm, &rank);

char* buf = NULL;
unsigned int recv_count;
int recv_count;
if (rank == (int)root)
{
std::stringstream s(std::ios_base::out | std::ios_base::binary);
Expand All @@ -144,7 +144,11 @@ template<typename T> void bcast(T& val, unsigned int root, const MPI_Comm mpi_co

// copy string to send buffer
std::string str = s.str();
recv_count = (unsigned int)str.size();
if (str.length() > std::numeric_limits<int>::max())
{
throw std::runtime_error("Serialized bytes overflow MPI limit");
}
recv_count = static_cast<int>(str.size());
buf = new char[recv_count];
str.copy(buf, recv_count);
}
Expand Down Expand Up @@ -182,7 +186,7 @@ void scatter_v(const std::vector<T>& in_values,

assert(in_values.size() == (unsigned int)size);

unsigned int recv_count;
int recv_count;
int* send_counts = NULL;
int* displs = NULL;

Expand All @@ -194,7 +198,7 @@ void scatter_v(const std::vector<T>& in_values,
// construct a vector of serialized objects
typename std::vector<T>::const_iterator it;
std::vector<std::string> str;
unsigned int len = 0;
size_t len = 0;
for (it = in_values.begin(); it != in_values.end(); ++it)
{
unsigned int idx = (unsigned int)(it - in_values.begin());
Expand All @@ -207,7 +211,11 @@ void scatter_v(const std::vector<T>& in_values,
str.push_back(s.str());

displs[idx] = (idx > 0) ? displs[idx - 1] + send_counts[idx - 1] : 0;
send_counts[idx] = (unsigned int)(str[idx].length());
if (str[idx].length() > std::numeric_limits<int>::max())
{
throw std::runtime_error("Serialized bytes overflow MPI limit");
}
send_counts[idx] = static_cast<int>(str[idx].length());
len += send_counts[idx];
}

Expand Down Expand Up @@ -262,7 +270,11 @@ void gather_v(const T& in_value,

// copy into send buffer
std::string str = s.str();
unsigned int send_count = (unsigned int)str.length();
if (str.length() > std::numeric_limits<int>::max())
{
throw std::runtime_error("Serialized bytes overflow MPI limit");
}
int send_count = static_cast<int>(str.length());

int* recv_counts = NULL;
int* displs = NULL;
Expand All @@ -279,7 +291,7 @@ void gather_v(const T& in_value,
char* rbuf = NULL;
if (rank == (int)root)
{
unsigned int len = 0;
size_t len = 0;
for (unsigned int i = 0; i < (unsigned int)size; i++)
{
displs[i] = (i > 0) ? displs[i - 1] + recv_counts[i - 1] : 0;
Expand Down Expand Up @@ -335,7 +347,11 @@ void all_gather_v(const T& in_value, std::vector<T>& out_values, const MPI_Comm

// copy into send buffer
std::string str = s.str();
unsigned int send_count = (unsigned int)str.length();
if (str.length() > std::numeric_limits<int>::max())
{
throw std::runtime_error("Serialized bytes overflow MPI limit");
}
int send_count = static_cast<int>(str.length());

// allocate memory for buffer lengths
out_values.resize(size);
Expand All @@ -346,7 +362,7 @@ void all_gather_v(const T& in_value, std::vector<T>& out_values, const MPI_Comm
MPI_Allgather(&send_count, 1, MPI_INT, recv_counts, 1, MPI_INT, mpi_comm);

// allocate receiver buffer
unsigned int len = 0;
size_t len = 0;
for (unsigned int i = 0; i < (unsigned int)size; i++)
{
displs[i] = (i > 0) ? displs[i - 1] + recv_counts[i - 1] : 0;
Expand Down Expand Up @@ -400,7 +416,11 @@ template<typename T> void send(const T& val, const unsigned int dest, const MPI_

// copy string to send buffer
std::string str = s.str();
recv_count = (unsigned int)str.size();
if (str.length() > std::numeric_limits<int>::max())
{
throw std::runtime_error("Serialized bytes overflow MPI limit");
}
recv_count = static_cast<int>(str.size());
buf = new char[recv_count];
str.copy(buf, recv_count);

Expand Down
87 changes: 87 additions & 0 deletions hoomd/MPIConfiguration.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

#include "MPIConfiguration.h"

#include "VectorMath.h"

#ifdef ENABLE_MPI
#include "HOOMDMPI.h"
#endif
Expand Down Expand Up @@ -36,9 +38,72 @@ MPIConfiguration::MPIConfiguration(
int rank;
MPI_Comm_rank(m_mpi_comm, &rank);
m_rank = rank;

// create scalar2 data type for MPI
{
int blocklengths[] = {1, 1};
MPI_Datatype types[] = {MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR};
MPI_Aint offsets[] = {offsetof(Scalar2, x), offsetof(Scalar2, y)};

MPI_Datatype tmp;
MPI_Type_create_struct(2, blocklengths, offsets, types, &tmp);
MPI_Type_create_resized(tmp, 0, sizeof(Scalar2), &m_mpi_scalar2);
MPI_Type_commit(&m_mpi_scalar2);
}

// create scalar3 data type for MPI
{
int blocklengths[] = {1, 1, 1};
MPI_Datatype types[] = {MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR};
MPI_Aint offsets[] = {offsetof(Scalar3, x), offsetof(Scalar3, y), offsetof(Scalar3, z)};

MPI_Datatype tmp;
MPI_Type_create_struct(3, blocklengths, offsets, types, &tmp);
MPI_Type_create_resized(tmp, 0, sizeof(Scalar3), &m_mpi_scalar3);
MPI_Type_commit(&m_mpi_scalar3);
}

// create vec3<Scalar> data type for MPI
{
int blocklengths[] = {1, 1, 1};
MPI_Datatype types[] = {MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR};
MPI_Aint offsets[]
= {offsetof(vec3<Scalar>, x), offsetof(vec3<Scalar>, y), offsetof(vec3<Scalar>, z)};

MPI_Datatype tmp;
MPI_Type_create_struct(3, blocklengths, offsets, types, &tmp);
MPI_Type_create_resized(tmp, 0, sizeof(vec3<Scalar>), &m_mpi_vec3_scalar);
MPI_Type_commit(&m_mpi_vec3_scalar);
}

// create scalar4 data type for MPI
{
int blocklengths[] = {1, 1, 1, 1};
MPI_Datatype types[]
= {MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR, MPI_HOOMD_SCALAR};
MPI_Aint offsets[] = {offsetof(Scalar4, x),
offsetof(Scalar4, y),
offsetof(Scalar4, z),
offsetof(Scalar4, w)};

MPI_Datatype tmp;
MPI_Type_create_struct(4, blocklengths, offsets, types, &tmp);
MPI_Type_create_resized(tmp, 0, sizeof(Scalar4), &m_mpi_scalar4);
MPI_Type_commit(&m_mpi_scalar4);
}
#endif
}

MPIConfiguration::~MPIConfiguration()
{
#ifdef ENABLE_MPI
MPI_Type_free(&m_mpi_scalar2);
MPI_Type_free(&m_mpi_scalar3);
MPI_Type_free(&m_mpi_vec3_scalar);
MPI_Type_free(&m_mpi_scalar4);
#endif // ENABLE_MPI
}

void MPIConfiguration::splitPartitions(unsigned int nrank)
{
#ifdef ENABLE_MPI
Expand Down Expand Up @@ -82,6 +147,28 @@ unsigned int MPIConfiguration::getNRanks() const
#endif
}

#ifdef ENABLE_MPI
MPI_Datatype MPIConfiguration::getScalar2Datatype() const
{
return m_mpi_scalar2;
}

MPI_Datatype MPIConfiguration::getScalar3Datatype() const
{
return m_mpi_scalar3;
}

MPI_Datatype MPIConfiguration::getVec3ScalarDatatype() const
{
return m_mpi_vec3_scalar;
}

MPI_Datatype MPIConfiguration::getScalar4Datatype() const
{
return m_mpi_scalar4;
}
#endif // ENABLE_MPI

namespace detail
{
void export_MPIConfiguration(pybind11::module& m)
Expand Down
21 changes: 20 additions & 1 deletion hoomd/MPIConfiguration.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class PYBIND11_EXPORT MPIConfiguration
);

//! Destructor
virtual ~MPIConfiguration() { };
virtual ~MPIConfiguration();

#ifdef ENABLE_MPI
MPI_Comm operator()() const
Expand Down Expand Up @@ -139,10 +139,29 @@ class PYBIND11_EXPORT MPIConfiguration
return walltime;
}

#ifdef ENABLE_MPI
//! Get Scalar2 datatype
MPI_Datatype getScalar2Datatype() const;

//! Get Scalar3 datatype
MPI_Datatype getScalar3Datatype() const;

//! Get vec3<Scalar> datatype
MPI_Datatype getVec3ScalarDatatype() const;

//! Get Scalar4 datatype
MPI_Datatype getScalar4Datatype() const;
#endif // ENABLE_MPI

protected:
#ifdef ENABLE_MPI
MPI_Comm m_mpi_comm; //!< The MPI communicator
MPI_Comm m_hoomd_world; //!< The HOOMD world communicator

MPI_Datatype m_mpi_scalar2; //!< HOOMD Scalar2 MPI datatype
MPI_Datatype m_mpi_scalar3; //!< HOOMD Scalar3 MPI datatype
MPI_Datatype m_mpi_vec3_scalar; //!< HOOMD vec3<Scalar> MPI datatype
MPI_Datatype m_mpi_scalar4; //!< HOOMD Scalar4 MPI datatype
#endif
unsigned int m_rank; //!< Rank of this processor (0 if running in single-processor mode)
unsigned int m_n_rank; //!< Ranks per partition
Expand Down
10 changes: 6 additions & 4 deletions hoomd/SnapshotSystemData.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ void SnapshotSystemData<Real>::broadcast(unsigned int root,
{
#ifdef ENABLE_MPI
auto communicator = exec_conf->getMPICommunicator();
broadcast_box(exec_conf->getMPIConfig());
auto mpi_config = exec_conf->getMPIConfig();
broadcast_box(mpi_config);
if (exec_conf->getNRanks() > 1)
{
particle_data.bcast(root, communicator);
Expand All @@ -105,7 +106,7 @@ void SnapshotSystemData<Real>::broadcast(unsigned int root,
constraint_data.bcast(root, communicator);
pair_data.bcast(root, communicator);
#ifdef BUILD_MPCD
mpcd_data.bcast(root, communicator);
mpcd_data.bcast(root, communicator, mpi_config);
#endif
}
#endif
Expand All @@ -117,9 +118,10 @@ void SnapshotSystemData<Real>::broadcast_all(unsigned int root,
{
#ifdef ENABLE_MPI
MPI_Comm hoomd_world = exec_conf->getHOOMDWorldMPICommunicator();
auto mpi_config = exec_conf->getMPIConfig();
int n_ranks;
MPI_Comm_size(hoomd_world, &n_ranks);
broadcast_box(exec_conf->getMPIConfig());
broadcast_box(mpi_config);
if (n_ranks > 0)
{
particle_data.bcast(root, hoomd_world);
Expand All @@ -130,7 +132,7 @@ void SnapshotSystemData<Real>::broadcast_all(unsigned int root,
constraint_data.bcast(root, hoomd_world);
pair_data.bcast(root, hoomd_world);
#ifdef BUILD_MPCD
mpcd_data.bcast(root, hoomd_world);
mpcd_data.bcast(root, hoomd_world, mpi_config);
#endif
}
#endif
Expand Down
Loading