diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8cd6d89343a..b23f7e229c6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,8 +12,8 @@ /BASELINE_NOTES.md /inputFiles/ @CusiniM @cssherman @jhuang2601 @rrsettgast /src/coreComponents/LvArray @rrsettgast @wrtobin @corbett5 @CusiniM -/src/coreComponents/codingUtilities @rrsettgast @untereiner @corbett5 -/src/coreComponents/common @rrsettgast @MelReyCG @corbett5 +/src/coreComponents/codingUtilities @rrsettgast @untereiner @corbett5 @wrtobin +/src/coreComponents/common @rrsettgast @MelReyCG @corbett5 @wrtobin /src/coreComponents/constitutive @rrsettgast @CusiniM @dkachuma @paveltomin @joshua-white /src/coreComponents/dataRepository @rrsettgast @wrtobin @corbett5 /src/coreComponents/denseLinearAlgebra @rrsettgast @CusiniM @castelletto1 diff --git a/src/coreComponents/common/Logger.hpp b/src/coreComponents/common/Logger.hpp index 7246173f496..75ac08d2375 100644 --- a/src/coreComponents/common/Logger.hpp +++ b/src/coreComponents/common/Logger.hpp @@ -45,6 +45,25 @@ */ #define GEOS_LOG_VAR( ... ) LVARRAY_LOG_VAR( __VA_ARGS__ ) + +/** + * @brief Conditionally log a message. + * @param EXP an expression that will be evaluated as a predicate + * @param msg a message to log (any expression that can be stream inserted) + */ +#if defined(GEOS_DEVICE_COMPILE) +#define GEOS_LOG_IF( EXP, msg ) +#else +#define GEOS_LOG_IF( EXP, msg ) \ + do { \ + if( EXP ) \ + { \ + std::cout<< msg << std::endl; \ + } \ + } while( false ) +#endif + + /** * @brief Conditionally log a message on screen on rank 0. * @param EXP an expression that will be evaluated as a predicate @@ -438,7 +457,7 @@ * @param[in] minLevel minimum log level * @param[in] msg a message to log (any expression that can be stream inserted) */ -#define GEOS_LOG_LEVEL( minLevel, msg ) GEOS_INFO_IF( this->getLogLevel() >= minLevel, msg ); +#define GEOS_LOG_LEVEL( minLevel, msg ) GEOS_LOG_IF( this->getLogLevel() >= minLevel, msg ); /** * @brief Output messages (only on rank 0) based on current Group's log level. diff --git a/src/coreComponents/mainInterface/ProblemManager.cpp b/src/coreComponents/mainInterface/ProblemManager.cpp index 88b92fa57d1..bd6852fafaf 100644 --- a/src/coreComponents/mainInterface/ProblemManager.cpp +++ b/src/coreComponents/mainInterface/ProblemManager.cpp @@ -666,6 +666,7 @@ void ProblemManager::generateMesh() } domain.setupCommunications( useNonblockingMPI ); + domain.outputPartitionInformation(); domain.forMeshBodies( [&]( MeshBody & meshBody ) { diff --git a/src/coreComponents/mesh/DomainPartition.cpp b/src/coreComponents/mesh/DomainPartition.cpp index 68811964e66..9016bbd2b5f 100644 --- a/src/coreComponents/mesh/DomainPartition.cpp +++ b/src/coreComponents/mesh/DomainPartition.cpp @@ -27,7 +27,6 @@ #include "mesh/mpiCommunications/SpatialPartition.hpp" - namespace geos { using namespace dataRepository; @@ -321,4 +320,161 @@ void DomainPartition::addNeighbors( const unsigned int idim, } } +void DomainPartition::outputPartitionInformation() const +{ + + auto numberOfEntities = []( ObjectManagerBase const & objectManager ) + { + return std::make_pair( objectManager.getNumberOfLocalIndices(), objectManager.getNumberOfGhosts() ); + }; + + auto addCommaSeparators = []( localIndex const num ) + { + std::string const numStr = std::to_string( num ); + std::string result; + for( std::size_t i = 0; i < numStr.size(); ++i ) + { + result += numStr[i]; + if( ( numStr.size() - i - 1 ) % 3 == 0 && i != numStr.size() - 1 ) + { + result += ","; + } + } + return result; + }; + + GEOS_LOG_RANK_0( "MPI Partition information:" ); + + + forMeshBodies( [&]( MeshBody const & meshBody ) + { + meshBody.getMeshLevels().forSubGroupsIndex< MeshLevel >( [&]( int const level, MeshLevel const & meshLevel ) + { + if( level!=0 ) + { + + // get the number of local and ghost entities for each type + auto const [ numLocalNodes, numGhostNodes ] = numberOfEntities( meshLevel.getNodeManager() ); + real64 const nodeRatio = ( numLocalNodes + numGhostNodes ) > 0 ? real64( numLocalNodes ) / real64( numLocalNodes + numGhostNodes ) : -1.0; + auto const [ numLocalEdges, numGhostEdges ] = numberOfEntities( meshLevel.getEdgeManager() ); + real64 const edgeRatio = ( numLocalEdges + numGhostEdges ) > 0 ? real64( numLocalEdges ) / real64( numLocalEdges + numGhostEdges ) : -1.0; + auto const [ numLocalFaces, numGhostFaces ] = numberOfEntities( meshLevel.getFaceManager() ); + real64 const faceRatio = ( numLocalFaces + numGhostFaces ) > 0 ? real64( numLocalFaces ) / real64( numLocalFaces + numGhostFaces ) : -1.0; + + localIndex numLocalElems = 0; + localIndex numGhostElems = 0; + meshLevel.getElemManager().forElementSubRegions< CellElementSubRegion >( [&]( CellElementSubRegion const & subRegion ) + { + auto [ numLocalElemsInSubRegion, numGhostElemsInSubRegion ] = numberOfEntities( subRegion ); + numLocalElems += numLocalElemsInSubRegion; + numGhostElems += numGhostElemsInSubRegion; + } ); + real64 const elemRatio = ( numLocalElems + numGhostElems ) > 0 ? real64( numLocalElems ) / real64( numLocalElems + numGhostElems ) : -1.0; + + localIndex const values[8] = { numLocalNodes, numGhostNodes, numLocalEdges, numGhostEdges, numLocalFaces, numGhostFaces, numLocalElems, numGhostElems }; + localIndex minValues[8] = {0}; + localIndex maxValues[8] = {0}; + MpiWrapper::allReduce( values, minValues, 8, MPI_MIN, MPI_COMM_WORLD ); + MpiWrapper::allReduce( values, maxValues, 8, MPI_MAX, MPI_COMM_WORLD ); + localIndex const minNumLocalNodes = minValues[0]; + localIndex const maxNumLocalNodes = maxValues[0]; + localIndex const minNumGhostNodes = minValues[1]; + localIndex const maxNumGhostNodes = maxValues[1]; + localIndex const minNumLocalEdges = minValues[2]; + localIndex const maxNumLocalEdges = maxValues[2]; + localIndex const minNumGhostEdges = minValues[3]; + localIndex const maxNumGhostEdges = maxValues[3]; + localIndex const minNumLocalFaces = minValues[4]; + localIndex const maxNumLocalFaces = maxValues[4]; + localIndex const minNumGhostFaces = minValues[5]; + localIndex const maxNumGhostFaces = maxValues[5]; + localIndex const minNumLocalElems = minValues[6]; + localIndex const maxNumLocalElems = maxValues[6]; + localIndex const minNumGhostElems = minValues[7]; + localIndex const maxNumGhostElems = maxValues[7]; + + real64 const ratios[4] = { nodeRatio, edgeRatio, faceRatio, elemRatio }; + real64 minRatios[4] = {0}; + real64 maxRatios[4] = {0}; + MpiWrapper::allReduce( ratios, minRatios, 4, MPI_MIN, MPI_COMM_WORLD ); + MpiWrapper::allReduce( ratios, maxRatios, 4, MPI_MAX, MPI_COMM_WORLD ); + real64 const minNodeRatio = minRatios[0]; + real64 const maxNodeRatio = maxRatios[0]; + real64 const minEdgeRatio = minRatios[1]; + real64 const maxEdgeRatio = maxRatios[1]; + real64 const minFaceRatio = minRatios[2]; + real64 const maxFaceRatio = maxRatios[2]; + real64 const minElemRatio = minRatios[3]; + real64 const maxElemRatio = maxRatios[3]; + + GEOS_LOG_RANK_0( " MeshBody: " + meshBody.getName() + " MeshLevel: " + meshLevel.getName() + "\n" ); + GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" ); + GEOS_LOG_RANK_0( " | | Nodes | Edges | Faces | Elems |" ); + GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" ); + GEOS_LOG_RANK_0( GEOS_FMT( " |min(local/total)| {:4.2f} | {:4.2f} | {:4.2f} | {:4.2f} | ", + minNodeRatio, + minEdgeRatio, + minFaceRatio, + minElemRatio ) ); + GEOS_LOG_RANK_0( GEOS_FMT( " |max(local/total)| {:4.2f} | {:4.2f} | {:4.2f} | {:4.2f} | ", + maxNodeRatio, + maxEdgeRatio, + maxFaceRatio, + maxElemRatio ) ); + GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" ); + GEOS_LOG_RANK_0( " | Rank | local | ghost | local | ghost | local | ghost | local | ghost |" ); + GEOS_LOG_RANK_0( " |----------------|---------------|---------------|---------------|---------------|---------------|---------------|---------------|---------------|" ); + + + GEOS_LOG_RANK_0( GEOS_FMT( " | min | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} |", + addCommaSeparators( minNumLocalNodes ), + addCommaSeparators( minNumGhostNodes ), + addCommaSeparators( minNumLocalEdges ), + addCommaSeparators( minNumGhostEdges ), + addCommaSeparators( minNumLocalFaces ), + addCommaSeparators( minNumGhostFaces ), + addCommaSeparators( minNumLocalElems ), + addCommaSeparators( minNumGhostElems ) ) ); + + GEOS_LOG_RANK_0( GEOS_FMT( " | max | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} |", + addCommaSeparators( maxNumLocalNodes ), + addCommaSeparators( maxNumGhostNodes ), + addCommaSeparators( maxNumLocalEdges ), + addCommaSeparators( maxNumGhostEdges ), + addCommaSeparators( maxNumLocalFaces ), + addCommaSeparators( maxNumGhostFaces ), + addCommaSeparators( maxNumLocalElems ), + addCommaSeparators( maxNumGhostElems ) ) ); + + GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" ); + + // output in rank order + int const thisRank = MpiWrapper::commRank(); + for( int rank=0; rank13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | ", + rank, + addCommaSeparators( numLocalNodes ), + addCommaSeparators( numGhostNodes ), + addCommaSeparators( numLocalEdges ), + addCommaSeparators( numGhostEdges ), + addCommaSeparators( numLocalFaces ), + addCommaSeparators( numGhostFaces ), + addCommaSeparators( numLocalElems ), + addCommaSeparators( numGhostElems ) ) ); + } + MpiWrapper::barrier(); + } + MpiWrapper::barrier(); + GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" ); + } + } ); + } + + ); + +} + } /* namespace geos */ diff --git a/src/coreComponents/mesh/DomainPartition.hpp b/src/coreComponents/mesh/DomainPartition.hpp index 27cea17a2f4..61a70ee0cba 100644 --- a/src/coreComponents/mesh/DomainPartition.hpp +++ b/src/coreComponents/mesh/DomainPartition.hpp @@ -116,6 +116,12 @@ class DomainPartition : public dataRepository::Group void addNeighbors( const unsigned int idim, MPI_Comm & cartcomm, int * ncoords ); + + /** + * @brief Outputs information about the partitioning of the domain. + */ + void outputPartitionInformation() const; + ///@}