Skip to content

Commit

Permalink
Comments/Documentation for New Ghosting Architecture (#3189)
Browse files Browse the repository at this point in the history 
* added comments for global numbering (Ryan's version)

* comments for constructing owned and present mesh data (Ryan's version)

* comments for construction of alocal djacency matrix data by each rank (Ryan's version)

* Comments through ghostingFootprint (Ryan's version)

* Comments through construction of GhostSend/GhostRecv for each rank (Ryan's version)

* complete comments for performGhosting (Ryan's version)

* Update comments about sorting following fix

* finished comments of doTheNewGhosting (Ryan's version)

* make new ghosting active/deactive by setting command line flag (-g 1)
  • Loading branch information
@ryar9534
ryar9534 authored Jul 3, 2024
1 parent cb01dc9 commit 7058cb3
Show file tree
Hide file tree
Showing 7 changed files with 902 additions and 528 deletions.
3 changes: 3 additions & 0 deletions src/coreComponents/common/initializeEnvironment.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ struct CommandLineOptions

/// Print memory usage in data repository
real64 printMemoryUsage = -1.0;

/// Use new ghosting strategy
integer useNewGhosting = 0;
};

/**
Expand Down
20 changes: 11 additions & 9 deletions src/coreComponents/mainInterface/ProblemManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,11 @@ ProblemManager::ProblemManager( conduit::Node & root ):
setRestartFlags( RestartFlags::WRITE ).
setDescription( "Whether to disallow using pinned memory allocations for MPI communication buffers." );

m_useNewGhosting = 1;
// registerWrapper( viewKeysStruct::useNewGhostingString(), &m_useNewGhosting ).
// setInputFlag( InputFlags::OPTIONAL ).
// setApplyDefaultValue( 0 ).
// setDescription( "Controls the use of the new ghosting implementation." );
commandLine.registerWrapper< integer >( viewKeys.useNewGhosting.key( ) ).
setApplyDefaultValue( 0 ).
setRestartFlags( RestartFlags::WRITE ).
setDescription( "Whether to use new ghosting strategy" );

}

ProblemManager::~ProblemManager()
Expand Down Expand Up @@ -191,6 +191,7 @@ void ProblemManager::parseCommandLineInput()
commandLine.getReference< integer >( viewKeys.overridePartitionNumbers ) = opts.overridePartitionNumbers;
commandLine.getReference< integer >( viewKeys.useNonblockingMPI ) = opts.useNonblockingMPI;
commandLine.getReference< integer >( viewKeys.suppressPinned ) = opts.suppressPinned;
commandLine.getReference< integer >( viewKeys.useNewGhosting ) = opts.useNewGhosting;

string & outputDirectory = commandLine.getReference< string >( viewKeys.outputDirectory );
outputDirectory = opts.outputDirectory;
Expand Down Expand Up @@ -633,10 +634,12 @@ void ProblemManager::generateMesh()
GEOS_MARK_FUNCTION;
DomainPartition & domain = getDomainPartition();

Group const & commandLine = getGroup< Group >( groupKeys.commandLine );

MeshManager & meshManager = this->getGroup< MeshManager >( groupKeys.meshManager );

GEOS_LOG_RANK( "m_useNewGhosting = " << m_useNewGhosting );
meshManager.generateMeshes( m_useNewGhosting, domain );
GEOS_LOG_RANK( "useNewGhosting = " << commandLine.getReference< integer >( viewKeys.useNewGhosting ) );
meshManager.generateMeshes( commandLine.getReference< integer >( viewKeys.useNewGhosting ), domain );

// get all the discretizations from the numerical methods.
// map< pair< mesh body name, pointer to discretization>, array of region names >
Expand All @@ -658,7 +661,7 @@ void ProblemManager::generateMesh()
}
else
{
if( m_useNewGhosting )
if( commandLine.getReference< integer >( viewKeys.useNewGhosting ) )
{
GEOS_LOG_RANK_0( "Generating the mesh levels for the new ghosting." );
generateMeshLevelFreeFct( meshBody.getMeshMappings(),
Expand All @@ -685,7 +688,6 @@ void ProblemManager::generateMesh()
}
} );

Group const & commandLine = this->getGroup< Group >( groupKeys.commandLine );
integer const useNonblockingMPI = commandLine.getReference< integer >( viewKeys.useNonblockingMPI );
// domain.setupBaseLevelMeshGlobalInfo();

Expand Down
7 changes: 1 addition & 6 deletions src/coreComponents/mainInterface/ProblemManager.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,9 +230,7 @@ class ProblemManager : public dataRepository::Group
dataRepository::ViewKey outputDirectory = {"outputDirectory"}; ///< Output directory key
dataRepository::ViewKey useNonblockingMPI = {"useNonblockingMPI"}; ///< Flag to use non-block MPI key
dataRepository::ViewKey suppressPinned = {"suppressPinned"}; ///< Flag to suppress use of pinned memory key
constexpr static char const * useNewGhostingString()
{ return "useNewGhosting"; }

dataRepository::ViewKey useNewGhosting = {"useNewGhosting"}; ///< Flag to use new ghosting strategies
} viewKeys; ///< Command line input viewKeys

/// Child group viewKeys
Expand Down Expand Up @@ -385,9 +383,6 @@ class ProblemManager : public dataRepository::Group

/// The FieldSpecificationManager
FieldSpecificationManager * m_fieldSpecificationManager;

/// Whether we should use the new ghosting implementation.
integer m_useNewGhosting = 0;
};

} /* namespace geos */
Expand Down
7 changes: 7 additions & 0 deletions src/coreComponents/mainInterface/initialization.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ std::unique_ptr< CommandLineOptions > parseCommandLineOptions( int argc, char *
TRACE_DATA_MIGRATION,
MEMORY_USAGE,
PAUSE_FOR,
NEW_GHOST,
};

const option::Descriptor usage[] =
Expand All @@ -123,6 +124,7 @@ std::unique_ptr< CommandLineOptions > parseCommandLineOptions( int argc, char *
{ TRACE_DATA_MIGRATION, 0, "", "trace-data-migration", Arg::None, "\t--trace-data-migration, \t Trace host-device data migration" },
{ MEMORY_USAGE, 0, "m", "memory-usage", Arg::nonEmpty, "\t-m, --memory-usage, \t Minimum threshold for printing out memory allocations in a member of the data repository." },
{ PAUSE_FOR, 0, "", "pause-for", Arg::numeric, "\t--pause-for, \t Pause geosx for a given number of seconds before starting execution" },
{ NEW_GHOST, 0, "g", "new-ghosting", Arg::numeric, "\t-g, --new-ghosting, \t Use new ghosting strategy" },
{ 0, 0, nullptr, nullptr, nullptr, nullptr }
};

Expand Down Expand Up @@ -237,6 +239,11 @@ std::unique_ptr< CommandLineOptions > parseCommandLineOptions( int argc, char *
std::this_thread::sleep_for( std::chrono::seconds( duration ) );
}
break;
case NEW_GHOST:
{
commandLineOptions->useNewGhosting = 1;
}
break;
}
}

Expand Down
37 changes: 36 additions & 1 deletion src/coreComponents/mesh/generators/BuildPods.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,14 @@ std::map< GI, toLocIdx_t< GI > > buildGlobalToLocalMap( std::set< GI > const & g
return g2l;
}

// Consolidate the mesh graphs on the current rank into one object
MeshGraph mergeMeshGraph( MeshGraph const & owned,
MeshGraph const & present,
MeshGraph const & ghosts )
{
// inititalize the consolidated version with owned
MeshGraph result{ owned };
// insert the present and ghost data
for( MeshGraph const & graph: { present, ghosts } )
{
result.c2f.insert( std::cbegin( graph.c2f ), std::cend( graph.c2f ) );
Expand Down Expand Up @@ -89,6 +92,7 @@ struct UpwardMappings
std::map< NodeLocIdx, std::vector< CellLocIdx > > n2c;
};

// converts generic map to arrayOfArrays
template< class T, class U >
ArrayOfArrays< localIndex > convertToAoA( std::map< T, std::vector< U > > const & t2u )
{
Expand Down Expand Up @@ -138,6 +142,7 @@ array2d< localIndex, P > convertToA2d( std::map< T, std::vector< U > > const & t
return t2u_;
}

// converts map to unordered map
template< class GI >
unordered_map< globalIndex, localIndex > convertGlobalToLocalMap( std::map< GI, toLocIdx_t< GI > > const & g2l )
{
Expand All @@ -150,6 +155,7 @@ unordered_map< globalIndex, localIndex > convertGlobalToLocalMap( std::map< GI,
return g2l_;
}

// removes MPIRank typed int, changes from std::vector to array1d
template< typename LI >
std::map< integer, array1d< localIndex > > toFlavorlessMapping( std::map< MpiRank, std::vector< LI > > const & input )
{
Expand All @@ -165,7 +171,7 @@ std::map< integer, array1d< localIndex > > toFlavorlessMapping( std::map< MpiRan
}

return output;
};
}

EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges,
std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n,
Expand Down Expand Up @@ -219,6 +225,8 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces,
toFlavorlessMapping( recv ) );
}

// Function to create a nodeManager from our local upward mapping data
// By flavorless, we are indicating that we are changing data-structures to GEOS ones, etc
NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes,
std::map< NodeGlbIdx, std::array< double, 3 > > const & n2pos,
std::map< NodeLocIdx, std::vector< EdgeLocIdx > > const & n2e,
Expand Down Expand Up @@ -282,6 +290,7 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l,
DownwardMappings res;

// Building the `e2n` (edges to nodes) mapping
// Simply looping through the meshgraph and converting to local indices
for( auto const & [egi, ngis]: graph.e2n )
{
NodeLocIdx const nli0 = g2l.nodes.at( std::get< 0 >( ngis ) );
Expand All @@ -290,8 +299,10 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l,
}

// Building the `f2n` (face to nodes) and `f2e` (faces to edges) mappings
// Loop through meshgraph
for( auto const & [fgi, edgeInfos]: graph.f2e )
{
// convert to face local index
FaceLocIdx const & fli = g2l.faces.at( fgi );

std::vector< NodeLocIdx > & nodes = res.f2n[fli];
Expand All @@ -300,18 +311,23 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l,
nodes.reserve( std::size( edgeInfos ) );
edges.reserve( std::size( edgeInfos ) );

// Loop over the edges connected to the face
for( EdgeInfo const & edgeInfo: edgeInfos )
{
// use the meshgraph again to go from edge to node, convert to local indices to fill face2node
std::tuple< NodeGlbIdx, NodeGlbIdx > const & ngis = graph.e2n.at( edgeInfo.index );
nodes.emplace_back( edgeInfo.start == 0 ? g2l.nodes.at( std::get< 0 >( ngis ) ) : g2l.nodes.at( std::get< 1 >( ngis ) ) );

// just convert edge global id to local to fill face2edge
edges.emplace_back( g2l.edges.at( edgeInfo.index ) );
}
}

// Building the `c2n` (cell to nodes), `c2e` (cell to edges) and `c2f` (cell to faces) mappings
// loop thorugh meshgraph cells2face
for( auto const & [cgi, faceInfos]: graph.c2f )
{
// convert to cell local index
CellLocIdx const & cli = g2l.cells.at( cgi );

std::vector< FaceLocIdx > & faces = res.c2f[cli];
Expand All @@ -321,12 +337,14 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l,
edges.reserve( std::size( faceInfos ) );

// c2f
// simple conversion of global to local index is all thats needed
for( FaceInfo const & faceInfo: faceInfos )
{
faces.emplace_back( g2l.faces.at( faceInfo.index ) );
}

// c2e
// loop over the faces in the cell and collect all their edges
std::set< EdgeLocIdx > tmpEdges;
for( FaceLocIdx const & fli: faces )
{
Expand All @@ -336,16 +354,19 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l,
edges.assign( std::cbegin( tmpEdges ), std::cend( tmpEdges ) );

// c2n
// Note how we are hard-coded for hexs
FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type.
FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7)

std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( g2l.faces.at( bottomFace.index ) );
std::vector< NodeLocIdx > const & topNodes = res.f2n.at( g2l.faces.at( topFace.index ) );

// reserFaceNodes resets the order using the flipped and start info
std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start );
std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start );

// TODO carefully check the ordering...
// looks like this is geos order
std::array< NodeLocIdx, 8 > const tmp{ bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] };
res.c2n[cli] = { tmp[0], tmp[1], tmp[3], tmp[2], tmp[4], tmp[5], tmp[7], tmp[6] };
}
Expand Down Expand Up @@ -494,18 +515,32 @@ void buildPods( MeshGraph const & owned,
GhostSend const & send,
MeshMappingImpl & meshMappings )
{
// start by merging the 3 mesh graphs for the different types of data on the graph into 1 struct
MeshGraph const graph = mergeMeshGraph( owned, present, ghosts );

// create GlobalToLocal, which maps global IDs to rank local IDs for (nodes, edges, faces, cells)
// mapKeys is just a utility which extracts keys
// buildGlobalToLocalMap just takes the global ids and maps them to 1, 2, 3, ...
GlobalToLocal const g2l{
buildGlobalToLocalMap( mapKeys< std::set >( graph.n2pos ) ),
buildGlobalToLocalMap( mapKeys< std::set >( graph.e2n ) ),
buildGlobalToLocalMap( mapKeys< std::set >( graph.f2e ) ),
buildGlobalToLocalMap( mapKeys< std::set >( graph.c2f ) )
};

// Now we build the mappings used by GEOS
// Downward - edges2nodes, faces2edges, faces2nodes, cells2faces, cells2edges, cells2nodes
// The difference is that these use local indices, thus we built g2l
// there are assumptions that we are working on hexs here
DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph );
// invert to downward mappings to get e2f, f2c, n2e, n2f, n2c
UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings );

// Now we can make our node/edge/face/cell manager
// NodeMgrImpl inherits from nodeManager (see pods.hpp) (interface with getters and setters)
// We basically have all the data, this function just casts it into the proper GEOS types
// same for edges, faces, cells

meshMappings.setEdgeMgr( makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ),
downwardMappings.e2n,
upwardMappings.e2f,
Expand Down
Loading

0 comments on commit 7058cb3

Please sign in to comment.