Skip to content

Commit

Permalink
Merge pull request #12611 from KratosMultiphysics/core/mpi/fix-hangin…
Browse files Browse the repository at this point in the history 
…g-node-submodelpart-paralellfillcommunicator

[Core][MPI] Adding new test for `ParallelFillCommunicator`
  • Loading branch information
@loumalouomega
loumalouomega authored Aug 6, 2024
2 parents c29d1dc + 42bfe52 commit be8876d
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 6 deletions.
63 changes: 59 additions & 4 deletions kratos/mpi/tests/cpp_tests/utilities/test_parallel_fill_communicator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,18 @@
#include "mpi/tests/test_utilities/mpi_cpp_test_utilities.h"
#include "mpi/utilities/parallel_fill_communicator.h"

namespace Kratos::Testing
namespace Kratos::Testing
{

KRATOS_TEST_CASE_IN_SUITE(ParallelFillCommunicatorExecute, KratosMPICoreFastSuite)
KRATOS_TEST_CASE_IN_SUITE(ParallelFillCommunicatorExecuteBarStructure, KratosMPICoreFastSuite)
{
// The model part
Model current_model;
ModelPart& r_model_part = current_model.CreateModelPart("Main");

// The data communicator
const DataCommunicator& r_data_communicator = Testing::GetDefaultDataCommunicator();

// MPI data
const int rank = r_data_communicator.Rank();
const int world_size = r_data_communicator.Size();
Expand Down Expand Up @@ -76,4 +76,59 @@ KRATOS_TEST_CASE_IN_SUITE(ParallelFillCommunicatorExecute, KratosMPICoreFastSuit
}
}

KRATOS_TEST_CASE_IN_SUITE(ParallelFillCommunicatorExecuteTriangleMesh, KratosMPICoreFastSuite)
{
// The model part
Model current_model;
ModelPart& r_model_part = current_model.CreateModelPart("Main");

// The data communicator
const DataCommunicator& r_data_communicator = Testing::GetDefaultDataCommunicator();

// MPI data
const int rank = r_data_communicator.Rank();
const int world_size = r_data_communicator.Size();

// Fill the model part
MPICppTestUtilities::GenerateDistributedTriangleMesh(r_model_part, r_data_communicator);

// Check that the communicator is correctly filled
const auto& r_neighbours_indices = r_model_part.GetCommunicator().NeighbourIndices();
if (world_size == 1) {
KRATOS_EXPECT_EQ(r_neighbours_indices.size(), 0);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().LocalMesh().NumberOfNodes(), 5);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().GhostMesh().NumberOfNodes(), 0);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().InterfaceMesh().NumberOfNodes(), 0);
} else if (world_size == 2) {
KRATOS_EXPECT_EQ(r_neighbours_indices.size(), 1);
if (rank == 0) {
KRATOS_EXPECT_EQ(r_neighbours_indices[0], 1);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().LocalMesh().NumberOfNodes(), 4);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().GhostMesh().NumberOfNodes(), 1);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().InterfaceMesh().NumberOfNodes(), 3);
} else if (rank == 1) {
KRATOS_EXPECT_EQ(r_neighbours_indices[0], 0);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().LocalMesh().NumberOfNodes(), 1);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().GhostMesh().NumberOfNodes(), 2);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().InterfaceMesh().NumberOfNodes(), 3);
}
} else {
if (rank == 0) {
KRATOS_EXPECT_EQ(r_neighbours_indices[0], 1);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().LocalMesh().NumberOfNodes(), 4);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().GhostMesh().NumberOfNodes(), 1);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().InterfaceMesh().NumberOfNodes(), 3);
} else if (rank == 1) {
KRATOS_EXPECT_EQ(r_neighbours_indices[0], 0);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().LocalMesh().NumberOfNodes(), 1);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().GhostMesh().NumberOfNodes(), 2);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().InterfaceMesh().NumberOfNodes(), 3);
} else { // The rest of the ranks
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().LocalMesh().NumberOfNodes(), 0);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().GhostMesh().NumberOfNodes(), 0);
KRATOS_EXPECT_EQ(r_model_part.GetCommunicator().InterfaceMesh().NumberOfNodes(), 0);
}
}
}

} // namespace Kratos::Testing
95 changes: 94 additions & 1 deletion kratos/mpi/tests/test_utilities/mpi_cpp_test_utilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@

// Project includes
#include "geometries/line_2d_2.h"
#include "geometries/triangle_2d_3.h"
#include "utilities/global_pointer_utilities.h"
#include "mpi/includes/mpi_data_communicator.h"
#include "mpi/tests/test_utilities/mpi_cpp_test_utilities.h"
#include "mpi/utilities/model_part_communicator_utilities.h"
#include "mpi/utilities/parallel_fill_communicator.h"
#include "tests/test_utilities/test_bar_element.h"
#include "processes/skin_detection_process.h"

namespace Kratos
{
Expand Down Expand Up @@ -172,7 +174,98 @@ void MPICppTestUtilities::GenerateDistributedBarStructure(
}
}

// Compute communicaton plan and fill communicator meshes correctly
// Compute communication plan and fill communicator meshes correctly
auto filler = ParallelFillCommunicator(rModelPart, rDataCommunicator);
filler.Execute();
}

/***********************************************************************************/
/***********************************************************************************/

void MPICppTestUtilities::GenerateDistributedTriangleMesh(
ModelPart& rModelPart,
const DataCommunicator& rDataCommunicator
)
{
// Add variables
rModelPart.AddNodalSolutionStepVariable(PARTITION_INDEX);

// Create properties
auto p_prop = rModelPart.CreateNewProperties(1, 0);

// MPI data
const int rank = rDataCommunicator.Rank();
const int world_size = rDataCommunicator.Size();

// Initially everything in one partition
if (world_size == 1) {
auto pnode1 = rModelPart.CreateNewNode(1, 0.0, 0.0, 0.0);
auto pnode2 = rModelPart.CreateNewNode(2, 1.0, 0.0, 0.0);
auto pnode3 = rModelPart.CreateNewNode(3, 1.0, 1.0, 0.0);
auto pnode4 = rModelPart.CreateNewNode(4, 0.0, 1.0, 0.0);
auto pnode5 = rModelPart.CreateNewNode(5, 0.5, 0.5, 0.0);

/// Add PARTITION_INDEX
for (auto& r_node : rModelPart.Nodes()) {
r_node.FastGetSolutionStepValue(PARTITION_INDEX) = rank;
}

auto pgeom1 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode1, pnode2, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 1, pgeom1, p_prop));
auto pgeom2 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode2, pnode3, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 2, pgeom2, p_prop));
auto pgeom3 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode3, pnode4, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 3, pgeom3, p_prop));
auto pgeom4 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode4, pnode1, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 4, pgeom4, p_prop));
} else { // if (world_size == 1) { // TODO: Do more than one partition
if (rank == 0) {
auto pnode1 = rModelPart.CreateNewNode(1, 0.0, 0.0, 0.0);
auto pnode2 = rModelPart.CreateNewNode(2, 1.0, 0.0, 0.0);
auto pnode3 = rModelPart.CreateNewNode(3, 1.0, 1.0, 0.0);
auto pnode4 = rModelPart.CreateNewNode(4, 0.0, 1.0, 0.0);
auto pnode5 = rModelPart.CreateNewNode(5, 0.5, 0.5, 0.0);

/// Add PARTITION_INDEX
for (auto& r_node : rModelPart.Nodes()) {
r_node.FastGetSolutionStepValue(PARTITION_INDEX) = 0;
}
pnode5->FastGetSolutionStepValue(PARTITION_INDEX) = 1;

auto pgeom1 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode1, pnode2, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 1, pgeom1, p_prop));
auto pgeom2 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode2, pnode3, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 2, pgeom2, p_prop));
auto pgeom4 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode4, pnode1, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 4, pgeom4, p_prop));
} else if (rank == 1) {
auto pnode3 = rModelPart.CreateNewNode(3, 1.0, 1.0, 0.0);
auto pnode4 = rModelPart.CreateNewNode(4, 0.0, 1.0, 0.0);
auto pnode5 = rModelPart.CreateNewNode(5, 0.5, 0.5, 0.0);

/// Add PARTITION_INDEX
for (auto& r_node : rModelPart.Nodes()) {
r_node.FastGetSolutionStepValue(PARTITION_INDEX) = 0;
}
pnode5->FastGetSolutionStepValue(PARTITION_INDEX) = 1;

auto pgeom3 = Kratos::make_shared<Triangle2D3<Node>>(PointerVector<Node>{std::vector<Node::Pointer>({pnode3, pnode4, pnode5})});
rModelPart.AddElement(Kratos::make_intrusive<Testing::TestBarElement>( 3, pgeom3, p_prop));
}
}

// Detect skin
Parameters skin_process_parameters = Parameters(R"(
{
"name_auxiliar_model_part" : "SkinModelPart",
"name_auxiliar_condition" : "Condition",
"list_model_parts_to_assign_conditions" : [],
"echo_level" : 0
})");
auto skin_process = SkinDetectionProcess<2>(rModelPart, skin_process_parameters);
skin_process.Execute();

// Compute communication plan and fill communicator meshes correctly
auto filler = ParallelFillCommunicator(rModelPart, rDataCommunicator);
filler.Execute();
}
Expand Down
10 changes: 10 additions & 0 deletions kratos/mpi/tests/test_utilities/mpi_cpp_test_utilities.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ static void GenerateDistributedBarStructure(
const DataCommunicator& rDataCommunicator
);

/**
* @brief It generates a triangle mesh
* @param rModelPart The model part to be filled
* @param rDataCommunicator The data communicator
*/
static void GenerateDistributedTriangleMesh(
ModelPart& rModelPart,
const DataCommunicator& rDataCommunicator
);

/**
* @brief Synchronizes local pointers to global pointers of the given ModelPart using the provided
* @details DataCommunicator and returns a shared pointer to a GlobalPointerCommunicator<Node>.
Expand Down
2 changes: 1 addition & 1 deletion kratos/mpi/utilities/parallel_fill_communicator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ void ParallelFillCommunicator::GenerateMeshes(
}

std::vector<int> ids_to_send;
{ // Syncronize how many nodes need to be sent/received.
{ // Synchronize how many nodes need to be sent/received.
int send_tag = Color;
int receive_tag = Color;
std::size_t recv_buf = r_data_communicator.SendRecv(ids_to_receive.size(), NeighbourPID, send_tag, NeighbourPID, receive_tag);
Expand Down

0 comments on commit be8876d

Please sign in to comment.