Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid calling METIS for cell partitioning with one task #129

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 52 additions & 18 deletions components/omega/src/base/Decomp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -425,29 +425,37 @@ Decomp::Decomp(
// Close file
Err = IO::closeFile(FileID);

// Use the mesh adjacency information to create a partition of cells
switch (Method) { // branch depending on method chosen
// In the case of single task avoid calling a full partitioning routine and
// just set the needed variables directly. This is done because some METIS
// functions can raise SIGFPE when numparts == 1 due to division by zero
// See: https://github.com/KarypisLab/METIS/issues/67
if (NumTasks == 1) {
partCellsSingleTask();
} else {
// Use the mesh adjacency information to create a partition of cells
switch (Method) { // branch depending on method chosen

//---------------------------------------------------------------------------
// ParMetis KWay method
case PartMethodMetisKWay: {
//---------------------------------------------------------------------------
// ParMetis KWay method
case PartMethodMetisKWay: {

Err = partCellsKWay(InEnv, CellsOnCellInit);
if (Err != 0) {
LOG_CRITICAL("Decomp: Error partitioning cells KWay");
return;
}
break;
} // end case MethodKWay
Err = partCellsKWay(InEnv, CellsOnCellInit);
if (Err != 0) {
LOG_CRITICAL("Decomp: Error partitioning cells KWay");
return;
}
break;
} // end case MethodKWay

//---------------------------------------------------------------------------
// Unknown partitioning method
//---------------------------------------------------------------------------
// Unknown partitioning method

default:
LOG_CRITICAL("Decomp: Unknown or unsupported decomposition method");
return;
default:
LOG_CRITICAL("Decomp: Unknown or unsupported decomposition method");
return;

} // End switch on Method
} // End switch on Method
}

//---------------------------------------------------------------------------

Expand Down Expand Up @@ -766,6 +774,32 @@ Decomp *Decomp::get(const std::string Name ///< [in] Name of environment

} // end get Decomposition

//------------------------------------------------------------------------------
// Trivially partition cells in the case of single task
// It sets the NCells sizes (NCellsOwned,
// NCellsHalo array, NCellsAll and NCellsSize) and the final CellID
// and CellLoc arrays
void Decomp::partCellsSingleTask() {

NCellsOwned = NCellsGlobal;

NCellsHaloH = HostArray1DI4("NCellsHalo", HaloWidth);
for (int Halo = 0; Halo < HaloWidth; ++Halo) {
NCellsHaloH(Halo) = NCellsGlobal;
}

NCellsAll = NCellsGlobal;
NCellsSize = NCellsGlobal + 1;

CellIDH = HostArray1DI4("CellID", NCellsSize);
CellLocH = HostArray2DI4("CellLoc", NCellsSize, 2);
for (int Cell = 0; Cell < NCellsAll; ++Cell) {
CellIDH(Cell) = Cell + 1;
CellLocH(Cell, 0) = 0;
CellLocH(Cell, 1) = Cell;
}
} // end function partCellsSingleTask

//------------------------------------------------------------------------------
// Partition the cells using the Metis/ParMetis KWay method
// After this partitioning, the decomposition class member CellID and
Expand Down
6 changes: 6 additions & 0 deletions components/omega/src/base/Decomp.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ class Decomp {
const std::vector<I4> &CellsOnCellInit ///< [in] cell nbrs in init dstrb
);

/// Trivially partition cells in the case of single task
/// It sets the NCells sizes (NCellsOwned,
/// NCellsHalo array, NCellsAll and NCellsSize) and the final CellID
/// and CellLoc arrays
void partCellsSingleTask();

/// Partition the edges given the cell partition and edge connectivity
/// The first cell ID associated with an edge in the CellsOnEdge array
/// is assumed to own the edge. The inputs are the edge-cell connectivity
Expand Down
21 changes: 16 additions & 5 deletions components/omega/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,24 @@ add_omega_test(
"-n;8"
)

#############
# Decomp test
#############
##########################
# Decomp test using 1 task
##########################

add_omega_test(
DECOMP_NTASK1_TEST
testDecompNTask1.exe
base/DecompTest.cpp
"-n;1"
)

###########################
# Decomp test using 8 tasks
###########################

add_omega_test(
DECOMP_TEST
testDecomp.exe
DECOMP_NTASK8_TEST
testDecompNTask8.exe
base/DecompTest.cpp
"-n;8"
)
Expand Down
Loading