Skip to content

Commit

Permalink
feat: Add partition information to terminal output (#3246)
Browse files Browse the repository at this point in the history 
* add function to output partition information

* add to CODEOWNERS file
  • Loading branch information
@rrsettgast
rrsettgast authored Jul 31, 2024
1 parent 2175cc4 commit c2fe0d3
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 4 deletions.
4 changes: 2 additions & 2 deletions .github/CODEOWNERS
View file
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
/BASELINE_NOTES.md
/inputFiles/ @CusiniM @cssherman @jhuang2601 @rrsettgast
/src/coreComponents/LvArray @rrsettgast @wrtobin @corbett5 @CusiniM
/src/coreComponents/codingUtilities @rrsettgast @untereiner @corbett5
/src/coreComponents/common @rrsettgast @MelReyCG @corbett5
/src/coreComponents/codingUtilities @rrsettgast @untereiner @corbett5 @wrtobin
/src/coreComponents/common @rrsettgast @MelReyCG @corbett5 @wrtobin
/src/coreComponents/constitutive @rrsettgast @CusiniM @dkachuma @paveltomin @joshua-white
/src/coreComponents/dataRepository @rrsettgast @wrtobin @corbett5
/src/coreComponents/denseLinearAlgebra @rrsettgast @CusiniM @castelletto1
Expand Down
21 changes: 20 additions & 1 deletion src/coreComponents/common/Logger.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,25 @@
*/
#define GEOS_LOG_VAR( ... ) LVARRAY_LOG_VAR( __VA_ARGS__ )


/**
* @brief Conditionally log a message.
* @param EXP an expression that will be evaluated as a predicate
* @param msg a message to log (any expression that can be stream inserted)
*/
#if defined(GEOS_DEVICE_COMPILE)
#define GEOS_LOG_IF( EXP, msg )
#else
#define GEOS_LOG_IF( EXP, msg ) \
do { \
if( EXP ) \
{ \
std::cout<< msg << std::endl; \
} \
} while( false )
#endif


/**
* @brief Conditionally log a message on screen on rank 0.
* @param EXP an expression that will be evaluated as a predicate
Expand Down Expand Up @@ -438,7 +457,7 @@
* @param[in] minLevel minimum log level
* @param[in] msg a message to log (any expression that can be stream inserted)
*/
#define GEOS_LOG_LEVEL( minLevel, msg ) GEOS_INFO_IF( this->getLogLevel() >= minLevel, msg );
#define GEOS_LOG_LEVEL( minLevel, msg ) GEOS_LOG_IF( this->getLogLevel() >= minLevel, msg );

/**
* @brief Output messages (only on rank 0) based on current Group's log level.
Expand Down
1 change: 1 addition & 0 deletions src/coreComponents/mainInterface/ProblemManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ void ProblemManager::generateMesh()
}

domain.setupCommunications( useNonblockingMPI );
domain.outputPartitionInformation();

domain.forMeshBodies( [&]( MeshBody & meshBody )
{
Expand Down
158 changes: 157 additions & 1 deletion src/coreComponents/mesh/DomainPartition.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
#include "mesh/mpiCommunications/SpatialPartition.hpp"



namespace geos
{
using namespace dataRepository;
Expand Down Expand Up @@ -321,4 +320,161 @@ void DomainPartition::addNeighbors( const unsigned int idim,
}
}

void DomainPartition::outputPartitionInformation() const
{

auto numberOfEntities = []( ObjectManagerBase const & objectManager )
{
return std::make_pair( objectManager.getNumberOfLocalIndices(), objectManager.getNumberOfGhosts() );
};

auto addCommaSeparators = []( localIndex const num )
{
std::string const numStr = std::to_string( num );
std::string result;
for( std::size_t i = 0; i < numStr.size(); ++i )
{
result += numStr[i];
if( ( numStr.size() - i - 1 ) % 3 == 0 && i != numStr.size() - 1 )
{
result += ",";
}
}
return result;
};

GEOS_LOG_RANK_0( "MPI Partition information:" );


forMeshBodies( [&]( MeshBody const & meshBody )
{
meshBody.getMeshLevels().forSubGroupsIndex< MeshLevel >( [&]( int const level, MeshLevel const & meshLevel )
{
if( level!=0 )
{

// get the number of local and ghost entities for each type
auto const [ numLocalNodes, numGhostNodes ] = numberOfEntities( meshLevel.getNodeManager() );
real64 const nodeRatio = ( numLocalNodes + numGhostNodes ) > 0 ? real64( numLocalNodes ) / real64( numLocalNodes + numGhostNodes ) : -1.0;
auto const [ numLocalEdges, numGhostEdges ] = numberOfEntities( meshLevel.getEdgeManager() );
real64 const edgeRatio = ( numLocalEdges + numGhostEdges ) > 0 ? real64( numLocalEdges ) / real64( numLocalEdges + numGhostEdges ) : -1.0;
auto const [ numLocalFaces, numGhostFaces ] = numberOfEntities( meshLevel.getFaceManager() );
real64 const faceRatio = ( numLocalFaces + numGhostFaces ) > 0 ? real64( numLocalFaces ) / real64( numLocalFaces + numGhostFaces ) : -1.0;

localIndex numLocalElems = 0;
localIndex numGhostElems = 0;
meshLevel.getElemManager().forElementSubRegions< CellElementSubRegion >( [&]( CellElementSubRegion const & subRegion )
{
auto [ numLocalElemsInSubRegion, numGhostElemsInSubRegion ] = numberOfEntities( subRegion );
numLocalElems += numLocalElemsInSubRegion;
numGhostElems += numGhostElemsInSubRegion;
} );
real64 const elemRatio = ( numLocalElems + numGhostElems ) > 0 ? real64( numLocalElems ) / real64( numLocalElems + numGhostElems ) : -1.0;

localIndex const values[8] = { numLocalNodes, numGhostNodes, numLocalEdges, numGhostEdges, numLocalFaces, numGhostFaces, numLocalElems, numGhostElems };
localIndex minValues[8] = {0};
localIndex maxValues[8] = {0};
MpiWrapper::allReduce( values, minValues, 8, MPI_MIN, MPI_COMM_WORLD );
MpiWrapper::allReduce( values, maxValues, 8, MPI_MAX, MPI_COMM_WORLD );
localIndex const minNumLocalNodes = minValues[0];
localIndex const maxNumLocalNodes = maxValues[0];
localIndex const minNumGhostNodes = minValues[1];
localIndex const maxNumGhostNodes = maxValues[1];
localIndex const minNumLocalEdges = minValues[2];
localIndex const maxNumLocalEdges = maxValues[2];
localIndex const minNumGhostEdges = minValues[3];
localIndex const maxNumGhostEdges = maxValues[3];
localIndex const minNumLocalFaces = minValues[4];
localIndex const maxNumLocalFaces = maxValues[4];
localIndex const minNumGhostFaces = minValues[5];
localIndex const maxNumGhostFaces = maxValues[5];
localIndex const minNumLocalElems = minValues[6];
localIndex const maxNumLocalElems = maxValues[6];
localIndex const minNumGhostElems = minValues[7];
localIndex const maxNumGhostElems = maxValues[7];

real64 const ratios[4] = { nodeRatio, edgeRatio, faceRatio, elemRatio };
real64 minRatios[4] = {0};
real64 maxRatios[4] = {0};
MpiWrapper::allReduce( ratios, minRatios, 4, MPI_MIN, MPI_COMM_WORLD );
MpiWrapper::allReduce( ratios, maxRatios, 4, MPI_MAX, MPI_COMM_WORLD );
real64 const minNodeRatio = minRatios[0];
real64 const maxNodeRatio = maxRatios[0];
real64 const minEdgeRatio = minRatios[1];
real64 const maxEdgeRatio = maxRatios[1];
real64 const minFaceRatio = minRatios[2];
real64 const maxFaceRatio = maxRatios[2];
real64 const minElemRatio = minRatios[3];
real64 const maxElemRatio = maxRatios[3];

GEOS_LOG_RANK_0( " MeshBody: " + meshBody.getName() + " MeshLevel: " + meshLevel.getName() + "\n" );
GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" );
GEOS_LOG_RANK_0( " | | Nodes | Edges | Faces | Elems |" );
GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" );
GEOS_LOG_RANK_0( GEOS_FMT( " |min(local/total)| {:4.2f} | {:4.2f} | {:4.2f} | {:4.2f} | ",
minNodeRatio,
minEdgeRatio,
minFaceRatio,
minElemRatio ) );
GEOS_LOG_RANK_0( GEOS_FMT( " |max(local/total)| {:4.2f} | {:4.2f} | {:4.2f} | {:4.2f} | ",
maxNodeRatio,
maxEdgeRatio,
maxFaceRatio,
maxElemRatio ) );
GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" );
GEOS_LOG_RANK_0( " | Rank | local | ghost | local | ghost | local | ghost | local | ghost |" );
GEOS_LOG_RANK_0( " |----------------|---------------|---------------|---------------|---------------|---------------|---------------|---------------|---------------|" );


GEOS_LOG_RANK_0( GEOS_FMT( " | min | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} |",
addCommaSeparators( minNumLocalNodes ),
addCommaSeparators( minNumGhostNodes ),
addCommaSeparators( minNumLocalEdges ),
addCommaSeparators( minNumGhostEdges ),
addCommaSeparators( minNumLocalFaces ),
addCommaSeparators( minNumGhostFaces ),
addCommaSeparators( minNumLocalElems ),
addCommaSeparators( minNumGhostElems ) ) );

GEOS_LOG_RANK_0( GEOS_FMT( " | max | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} |",
addCommaSeparators( maxNumLocalNodes ),
addCommaSeparators( maxNumGhostNodes ),
addCommaSeparators( maxNumLocalEdges ),
addCommaSeparators( maxNumGhostEdges ),
addCommaSeparators( maxNumLocalFaces ),
addCommaSeparators( maxNumGhostFaces ),
addCommaSeparators( maxNumLocalElems ),
addCommaSeparators( maxNumGhostElems ) ) );

GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" );

// output in rank order
int const thisRank = MpiWrapper::commRank();
for( int rank=0; rank<MpiWrapper::commSize(); ++rank )
{
if( rank == thisRank )
{
GEOS_LOG( GEOS_FMT( " | {:14} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | ",
rank,
addCommaSeparators( numLocalNodes ),
addCommaSeparators( numGhostNodes ),
addCommaSeparators( numLocalEdges ),
addCommaSeparators( numGhostEdges ),
addCommaSeparators( numLocalFaces ),
addCommaSeparators( numGhostFaces ),
addCommaSeparators( numLocalElems ),
addCommaSeparators( numGhostElems ) ) );
}
MpiWrapper::barrier();
}
MpiWrapper::barrier();
GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" );
}
} );
}

);

}

} /* namespace geos */
6 changes: 6 additions & 0 deletions src/coreComponents/mesh/DomainPartition.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,12 @@ class DomainPartition : public dataRepository::Group
void addNeighbors( const unsigned int idim,
MPI_Comm & cartcomm,
int * ncoords );

/**
* @brief Outputs information about the partitioning of the domain.
*/
void outputPartitionInformation() const;

///@}


Expand Down

0 comments on commit c2fe0d3

Please sign in to comment.