From 89729365d49d40197ea5c1bf88824d2a0529232d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:24:25 -0700 Subject: [PATCH 001/106] First dummy version that compiles. --- src/coreComponents/mesh/CMakeLists.txt | 2 + .../mesh/generators/MeshBlockMappings.cpp | 142 ++++++++++++++++++ .../mesh/generators/MeshBlockMappings.hpp | 103 +++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 src/coreComponents/mesh/generators/MeshBlockMappings.cpp create mode 100644 src/coreComponents/mesh/generators/MeshBlockMappings.hpp diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index 6065ef14dae..1fafa2eb0a6 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -54,6 +54,7 @@ set( mesh_headers generators/InternalMeshGenerator.hpp generators/InternalWellGenerator.hpp generators/InternalWellboreGenerator.hpp + generators/MeshBlockMappings.hpp generators/MeshGeneratorBase.hpp generators/ParMETISInterface.hpp generators/ParticleMeshGenerator.hpp @@ -127,6 +128,7 @@ set( mesh_sources generators/InternalMeshGenerator.cpp generators/InternalWellGenerator.cpp generators/InternalWellboreGenerator.cpp + generators/MeshBlockMappings.cpp generators/MeshGeneratorBase.cpp generators/ParMETISInterface.cpp generators/ParticleMeshGenerator.cpp diff --git a/src/coreComponents/mesh/generators/MeshBlockMappings.cpp b/src/coreComponents/mesh/generators/MeshBlockMappings.cpp new file mode 100644 index 00000000000..191071fc22e --- /dev/null +++ b/src/coreComponents/mesh/generators/MeshBlockMappings.cpp @@ -0,0 +1,142 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "MeshBlockMappings.hpp" + +namespace geos +{ + +using namespace dataRepository; + +MeshBlockMappings::MeshBlockMappings( string const & name, + Group * const parent ) + : + CellBlockManagerABC( name, parent ) +{ + this->registerGroup< Group >( viewKeyStruct::cellBlocks() ); + this->registerGroup< Group >( viewKeyStruct::faceBlocks() ); + this->registerGroup< Group >( viewKeyStruct::lineBlocks() ); +} + +Group & MeshBlockMappings::getCellBlocks() +{ + return this->getGroup( viewKeyStruct::cellBlocks() ); +} + +Group & MeshBlockMappings::getFaceBlocks() +{ + return this->getGroup( viewKeyStruct::faceBlocks() ); +} + +LineBlockABC const & MeshBlockMappings::getLineBlock( string name ) const +{ + return this->getGroup( viewKeyStruct::lineBlocks() ).getGroup< LineBlockABC >( name ); +} + +Group const & MeshBlockMappings::getCellBlocks() const +{ + return this->getGroup( viewKeyStruct::cellBlocks() ); +} + +Group const & MeshBlockMappings::getFaceBlocks() const +{ + return this->getGroup( viewKeyStruct::faceBlocks() ); +} + +localIndex MeshBlockMappings::numNodes() const +{ + return m_numNodes; +} + +localIndex MeshBlockMappings::numEdges() const +{ + return m_numEdges; +} + +localIndex MeshBlockMappings::numFaces() const +{ + return m_numFaces; +} + +array2d< real64, nodes::REFERENCE_POSITION_PERM > MeshBlockMappings::getNodePositions() const +{ + return {}; +} + +ArrayOfArrays< localIndex > MeshBlockMappings::getNodeToEdges() const +{ + return {}; +} + +ArrayOfArrays< localIndex > MeshBlockMappings::getNodeToFaces() const +{ + return {}; +} + +ToCellRelation< ArrayOfArrays< localIndex>> MeshBlockMappings::getNodeToElements() const +{ + return {}; +} + +array2d< localIndex > MeshBlockMappings::getEdgeToNodes() const +{ + return {}; +} + +ArrayOfArrays< localIndex > MeshBlockMappings::getEdgeToFaces() const +{ + return {}; +} + +ArrayOfArrays< localIndex > MeshBlockMappings::getFaceToNodes() const +{ + return {}; +} + +ArrayOfArrays< localIndex > MeshBlockMappings::getFaceToEdges() const +{ + return {}; +} + +ToCellRelation< array2d< localIndex>> MeshBlockMappings::getFaceToElements() const +{ + return {}; +} + +array1d< globalIndex > MeshBlockMappings::getNodeLocalToGlobal() const +{ + return {}; +} + +std::map< string, SortedArray< localIndex>> const & MeshBlockMappings::getNodeSets() const +{ + return m_nodeSets; +} + +real64 MeshBlockMappings::getGlobalLength() const +{ + return m_globalLength; +} + +void MeshBlockMappings::generateHighOrderMaps( localIndex const order, + globalIndex const maxVertexGlobalID, + globalIndex const maxEdgeGlobalID, + globalIndex const maxFaceGlobalID, + arrayView1d< globalIndex const > const edgeLocalToGlobal, + arrayView1d< globalIndex const > const faceLocalToGlobal ) +{ + GEOS_ERROR( "`MeshBlockMappings::generateHighOrderMaps` is not implemented and should not be called." ); +} + +} // geos \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/MeshBlockMappings.hpp b/src/coreComponents/mesh/generators/MeshBlockMappings.hpp new file mode 100644 index 00000000000..00fc352c020 --- /dev/null +++ b/src/coreComponents/mesh/generators/MeshBlockMappings.hpp @@ -0,0 +1,103 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_GENERATORS_MESHBLOCKMAPPINGS_HPP +#define GEOS_GENERATORS_MESHBLOCKMAPPINGS_HPP + +#include "CellBlockManagerABC.hpp" + +namespace geos +{ + +class MeshBlockMappings: public CellBlockManagerABC +{ +public: + MeshBlockMappings( string const & name, Group * const parent ); + + virtual Group & getCellBlocks() override; + + virtual Group & getFaceBlocks() override; + + virtual LineBlockABC const & getLineBlock( string name ) const override; + + virtual Group const & getCellBlocks() const override; + + virtual Group const & getFaceBlocks() const override; + + virtual localIndex numNodes() const override; + + virtual localIndex numEdges() const override; + + virtual localIndex numFaces() const override; + + virtual array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override; + + virtual ArrayOfArrays< localIndex > getNodeToEdges() const override; + + virtual ArrayOfArrays< localIndex > getNodeToFaces() const override; + + virtual ToCellRelation< ArrayOfArrays< localIndex>> getNodeToElements() const override; + + virtual array2d< localIndex > getEdgeToNodes() const override; + + virtual ArrayOfArrays< localIndex > getEdgeToFaces() const override; + + virtual ArrayOfArrays< localIndex > getFaceToNodes() const override; + + virtual ArrayOfArrays< localIndex > getFaceToEdges() const override; + + virtual ToCellRelation< array2d< localIndex>> getFaceToElements() const override; + + virtual array1d< globalIndex > getNodeLocalToGlobal() const override; + + virtual std::map< string, SortedArray< localIndex>> const & getNodeSets() const override; + + virtual real64 getGlobalLength() const override; + + virtual void generateHighOrderMaps( localIndex const order, + globalIndex const maxVertexGlobalID, + globalIndex const maxEdgeGlobalID, + globalIndex const maxFaceGlobalID, + arrayView1d< globalIndex const > const edgeLocalToGlobal, + arrayView1d< globalIndex const > const faceLocalToGlobal ) override; + +private: + + struct viewKeyStruct + { + /// Cell blocks key + static constexpr char const * cellBlocks() + { return "cellBlocks"; } + + /// Face blocks key + static constexpr char const * faceBlocks() + { return "faceBlocks"; } + + /// Line blocks key + static constexpr char const * lineBlocks() + { return "lineBlocks"; } + }; + + std::map< string, SortedArray< localIndex > > m_nodeSets; + + real64 m_globalLength; + + localIndex m_numNodes; + localIndex m_numFaces; + localIndex m_numEdges; +}; + +} // geos + +#endif //GEOS_GENERATORS_MESHBLOCKMAPPINGS_HPP From 095b262ee69d3c3ee9793395dac7c4f99a93cd46 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 14:53:57 -0700 Subject: [PATCH 002/106] v0 of the doc --- .../KeyComponents/Ghosting/ghosting.rst | 21 +++++ .../Ghosting/ghosting_algorithm.plantuml | 40 +++++++++ .../Ghosting/global_ghosting_setup.plantuml | 90 +++++++++++++++++++ .../Ghosting/global_numbering.plantuml | 46 ++++++++++ 4 files changed, 197 insertions(+) create mode 100644 src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst create mode 100644 src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting_algorithm.plantuml create mode 100644 src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_ghosting_setup.plantuml create mode 100644 src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_numbering.plantuml diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst new file mode 100644 index 00000000000..95133902259 --- /dev/null +++ b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst @@ -0,0 +1,21 @@ +.. _ghosting: + +Ghosting setup +###################### + +This describes the setup of the ghosting. + + +.. _global_ghosting_setup: +.. uml:: global_ghosting_setup.plantuml + :caption: The global parallel algorithm + + +.. _global_numbering: +.. uml:: global_numbering.plantuml + :caption: The global numbering algorithm + + +.. _ghosting_algorithm: +.. uml:: ghosting_algorithm.plantuml + :caption: The ghosting algorithm diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting_algorithm.plantuml b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting_algorithm.plantuml new file mode 100644 index 00000000000..6201b833620 --- /dev/null +++ b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting_algorithm.plantuml @@ -0,0 +1,40 @@ +@startuml + + + +participant "Rank k" as rk + +activate rk #WhiteSmoke + +rk -> rk++ #orange: Assemble paralllel sparse\nadjacency matrix A + +[<--> rk: Parallel matrix\nassembly (mpi) + +rk -> rk--++ #orange: Define rhs graph nodes support arrays\n(under the form of a parallel matrix) + +deactivate rk + +rk -\ rk-- <> + +loop halo size +rk -> rk++ #tomato: Compute ancestors using parallel\nmatrix-matrix multiplication +[<--> rk: matrix-matrix\nparallel multiplication (mpi) +rk -> rk--++ #tomato: Compute descendants using parallel\nmatrix-matrix multiplication +[<--> rk: matrix-matrix\nparallel multiplication (mpi) +||| +rk -\ rk-- <> +end + +rk -> rk++ #tomato: Update (add and remove)\nneighbor ranks +rk -> rk--++ #LimeGreen: Compute mappings\n(including ghosted information) +||| +||| + +@enduml diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_ghosting_setup.plantuml b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_ghosting_setup.plantuml new file mode 100644 index 00000000000..93c4911f182 --- /dev/null +++ b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_ghosting_setup.plantuml @@ -0,0 +1,90 @@ +@startuml +skinparam BoxPadding 10 +participant "Rank 0" as r0 +participant "..." as rtmp0 +box neighborhood of rank k #AliceBlue +participant "Rank k-1" as rkm1 +participant "Rank k" as rk +participant "Rank k+1" as rkp1 +end box +participant "..." as rtmpn +participant "Rank n" as rn + +==global numbering== + +activate r0 #WhiteSmoke +activate rtmp0 #WhiteSmoke +activate rkm1 #WhiteSmoke +activate rk #WhiteSmoke +activate rkp1 #WhiteSmoke +activate rtmpn #WhiteSmoke +activate rn #WhiteSmoke + +group Exchange meshes information ■ & \ncompute intersections ■ [mpi send/recv] + +!pragma teoz true + +r0 -[#tomato]->(90) rkm1++ #LimeGreen +& rkm1 -[#tomato]->(50) r0 +& r0 -[#tomato]->(80) rtmpn++ #LimeGreen +& rtmpn -[#tomato]->(100) r0++ #LimeGreen +& rtmpn -[#tomato]->(120) rn++ #LimeGreen +& rtmp0 -[#tomato]->(10) rkm1 +& rkm1 -[#tomato]->(40) rtmp0++ #LimeGreen +& rn -[#tomato]->(50) rtmpn +& rk -[#tomato]>(50) rkm1 +& rk -[#red]>(10) rkp1 +& rkm1 -[#red]>(40) rk +& rkm1 -[#red]>(50) rkp1++ #LimeGreen +& rkp1 -[#red]>(80) rk++ #LimeGreen +& rkp1 -[#red]>(70) rkm1glob + + +end group + +group Compute and exchange offsets ■, and number ■ [mpi scan] + +deactivate rtmp0 + +deactivate r0 +r0 -> r0++ #tomato +deactivate rkm1 +activate rtmp0 #tomato +r0 -[#red]>> rtmp0-- #tomato + +r0 -> r0++ #DeepSkyBlue + + +deactivate rn +deactivate rk + +activate rkm1 #tomato +rtmp0 -[#red]>> rkm1-- #tomato +deactivate rkp1 +rtmp0 -> rtmp0++ #DeepSkyBlue + + +activate rk #tomato +rkm1 -[#red]>> rk-- #tomato +rkm1 -> rkm1++ #DeepSkyBlue + +deactivate rtmpn + +activate rkp1 #tomato +rk -[#red]>> rkp1-- #tomato +rk -> rk++ #DeepSkyBlue + +activate rtmpn #tomato +rkp1 -[#red]>> rtmpn-- #tomato +rkp1 -> rkp1++ #DeepSkyBlue + +activate rn #tomato +rtmpn -[#red]>> rn-- #tomato +rtmpn -> rtmpn++ #DeepSkyBlue + +deactivate rn +rn -> rn++ #DeepSkyBlue + +end group + +@enduml diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_numbering.plantuml b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_numbering.plantuml new file mode 100644 index 00000000000..d1d5364d92a --- /dev/null +++ b/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_numbering.plantuml @@ -0,0 +1,46 @@ +@startuml + + + + +participant "Rank k" as rk + +activate rk #WhiteSmoke + +rk -> rk++ #LimeGreen: Format mesh information +deactivate rk + +rk -> rk++ #tomato: Exchange mesh information +[<--> rk: Communications with neighbor\n(mpi send/recv) +deactivate rk + +rk -> rk++ #LimeGreen: Compute intersections\nof edges and faces +rk -> rk++ #SpringGreen: Sort edges and faces\nfor efficient access +rk -> rk--++ #SpringGreen: Associate edges and faces\nto neighbor ranks +rk -> rk--++ #SpringGreen: Invert and sort associations +||| +rk -\ rk-- <> +rk -\ rk-- <> + +'deactivate rk + +[--> rk++ #tomato: Receive offsets\nfrom previous ranks (mpi scan) +rk -> rk++ #coral: Compute offsets +deactivate rk +rk -->]: Send updated offsets\nto next ranks (mpi scan) + +deactivate rk + +rk -> rk++ #DeepSkyBlue: Number edges and faces\nconsistently accros ranks + +||| +||| + +@enduml From 82184262528b81ed357e3098c8fa1fbaefeb18dc Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:14:59 -0700 Subject: [PATCH 003/106] Forgot toc... --- .../sphinx/developerGuide/KeyComponents/index_KeyComponents.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst b/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst index 6960fe15c24..4f3ec44f5c9 100644 --- a/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst +++ b/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst @@ -18,6 +18,8 @@ The main code components are described here. ./../../../../coreComponents/mesh/docs/meshDeveloperGuide + ./Ghosting/ghosting.rst + ./../../../../coreComponents/linearAlgebra/docs/DofManager.rst ./LvArray From fc3154f870ca31b24a440728a3c381352ebbf233 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 17:35:40 -0700 Subject: [PATCH 004/106] moved doc! --- .../mesh/docs}/Ghosting/ghosting.rst | 6 +++--- .../mesh/docs}/Ghosting/ghosting_algorithm.plantuml | 0 .../mesh/docs}/Ghosting/global_ghosting_setup.plantuml | 0 .../mesh/docs}/Ghosting/global_numbering.plantuml | 0 .../developerGuide/KeyComponents/index_KeyComponents.rst | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/{docs/sphinx/developerGuide/KeyComponents => coreComponents/mesh/docs}/Ghosting/ghosting.rst (57%) rename src/{docs/sphinx/developerGuide/KeyComponents => coreComponents/mesh/docs}/Ghosting/ghosting_algorithm.plantuml (100%) rename src/{docs/sphinx/developerGuide/KeyComponents => coreComponents/mesh/docs}/Ghosting/global_ghosting_setup.plantuml (100%) rename src/{docs/sphinx/developerGuide/KeyComponents => coreComponents/mesh/docs}/Ghosting/global_numbering.plantuml (100%) diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst b/src/coreComponents/mesh/docs/Ghosting/ghosting.rst similarity index 57% rename from src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst rename to src/coreComponents/mesh/docs/Ghosting/ghosting.rst index 95133902259..a3cdf6f5a93 100644 --- a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting.rst +++ b/src/coreComponents/mesh/docs/Ghosting/ghosting.rst @@ -7,15 +7,15 @@ This describes the setup of the ghosting. .. _global_ghosting_setup: -.. uml:: global_ghosting_setup.plantuml +.. uml:: /coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml :caption: The global parallel algorithm .. _global_numbering: -.. uml:: global_numbering.plantuml +.. uml:: /coreComponents/mesh/docs/Ghosting/global_numbering.plantuml :caption: The global numbering algorithm .. _ghosting_algorithm: -.. uml:: ghosting_algorithm.plantuml +.. uml:: /coreComponents/mesh/docs/Ghosting/ghosting_algorithm.plantuml :caption: The ghosting algorithm diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting_algorithm.plantuml b/src/coreComponents/mesh/docs/Ghosting/ghosting_algorithm.plantuml similarity index 100% rename from src/docs/sphinx/developerGuide/KeyComponents/Ghosting/ghosting_algorithm.plantuml rename to src/coreComponents/mesh/docs/Ghosting/ghosting_algorithm.plantuml diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_ghosting_setup.plantuml b/src/coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml similarity index 100% rename from src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_ghosting_setup.plantuml rename to src/coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml diff --git a/src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_numbering.plantuml b/src/coreComponents/mesh/docs/Ghosting/global_numbering.plantuml similarity index 100% rename from src/docs/sphinx/developerGuide/KeyComponents/Ghosting/global_numbering.plantuml rename to src/coreComponents/mesh/docs/Ghosting/global_numbering.plantuml diff --git a/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst b/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst index 4f3ec44f5c9..2fb4c31d9ac 100644 --- a/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst +++ b/src/docs/sphinx/developerGuide/KeyComponents/index_KeyComponents.rst @@ -18,7 +18,7 @@ The main code components are described here. ./../../../../coreComponents/mesh/docs/meshDeveloperGuide - ./Ghosting/ghosting.rst + ./../../../../coreComponents/mesh/docs/Ghosting/ghosting.rst ./../../../../coreComponents/linearAlgebra/docs/DofManager.rst From 44365e520203df4c8fb0a433ce78170b0fb63ad5 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:13:16 -0700 Subject: [PATCH 005/106] small plantuml test --- src/coreComponents/mesh/docs/Ghosting/ghosting.rst | 5 +++++ src/coreComponents/mesh/docs/Ghosting/test.plantuml | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 src/coreComponents/mesh/docs/Ghosting/test.plantuml diff --git a/src/coreComponents/mesh/docs/Ghosting/ghosting.rst b/src/coreComponents/mesh/docs/Ghosting/ghosting.rst index a3cdf6f5a93..126e28c67bc 100644 --- a/src/coreComponents/mesh/docs/Ghosting/ghosting.rst +++ b/src/coreComponents/mesh/docs/Ghosting/ghosting.rst @@ -6,6 +6,11 @@ Ghosting setup This describes the setup of the ghosting. +.. _plantuml_test: +.. uml:: /coreComponents/mesh/docs/Ghosting/test.plantuml + :caption: Some test + + .. _global_ghosting_setup: .. uml:: /coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml :caption: The global parallel algorithm diff --git a/src/coreComponents/mesh/docs/Ghosting/test.plantuml b/src/coreComponents/mesh/docs/Ghosting/test.plantuml new file mode 100644 index 00000000000..fad8541bb08 --- /dev/null +++ b/src/coreComponents/mesh/docs/Ghosting/test.plantuml @@ -0,0 +1,5 @@ +@startuml + +title Empty Diagram + +@enduml From 15b5cb31bb180e60bf24d73e6121c945203bdc50 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 19:05:05 -0700 Subject: [PATCH 006/106] update rtd machine --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index cac04680e9d..093ec654d50 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,7 +4,7 @@ version: 2 build: - os: "ubuntu-20.04" + os: "ubuntu-22.04" apt_packages: - npm - plantuml From 2f5c2f635c2e9502b3e8c4e17285ff62cf4d0612 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:06:02 -0700 Subject: [PATCH 007/106] update plantuml --- .readthedocs.yml | 5 ++++- src/conf.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 093ec654d50..b1c5ba8da4b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,8 +6,9 @@ version: 2 build: os: "ubuntu-22.04" apt_packages: + - curl - npm - - plantuml + - openjdk-21-jre-headless - texlive tools: nodejs: "16" @@ -16,6 +17,8 @@ build: post_install: - npm install -g npm@9.8.1 - npm install -g @mermaid-js/mermaid-cli@10.3.1 + # If you change the `/tmp/plantuml.jar` path, be sure to also update where this path is used as well (e.g. in `conf.py`). + - curl -fsSL https://github.com/plantuml/plantuml/releases/download/v1.2024.3/plantuml-gplv2-1.2024.3.jar -o /tmp/plantuml.jar # Set requirements to build the docs python: diff --git a/src/conf.py b/src/conf.py index 99989855bd3..f27f7843963 100644 --- a/src/conf.py +++ b/src/conf.py @@ -112,7 +112,7 @@ 'sphinxcontrib.programoutput' ] -plantuml = "/usr/bin/plantuml" +plantuml = "/usr/bin/java -Djava.awt.headless=true -jar /tmp/plantuml.jar" plantuml_output_format = "svg_img" plot_html_show_source_link = True From 177397a8a67c7aba339a6c6f6d2e5af380c1f9db Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:27:14 -0700 Subject: [PATCH 008/106] fix diagrams + remove tests --- src/coreComponents/mesh/docs/Ghosting/ghosting.rst | 5 ----- .../mesh/docs/Ghosting/global_ghosting_setup.plantuml | 4 +--- src/coreComponents/mesh/docs/Ghosting/test.plantuml | 5 ----- 3 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 src/coreComponents/mesh/docs/Ghosting/test.plantuml diff --git a/src/coreComponents/mesh/docs/Ghosting/ghosting.rst b/src/coreComponents/mesh/docs/Ghosting/ghosting.rst index 126e28c67bc..a3cdf6f5a93 100644 --- a/src/coreComponents/mesh/docs/Ghosting/ghosting.rst +++ b/src/coreComponents/mesh/docs/Ghosting/ghosting.rst @@ -6,11 +6,6 @@ Ghosting setup This describes the setup of the ghosting. -.. _plantuml_test: -.. uml:: /coreComponents/mesh/docs/Ghosting/test.plantuml - :caption: Some test - - .. _global_ghosting_setup: .. uml:: /coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml :caption: The global parallel algorithm diff --git a/src/coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml b/src/coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml index 93c4911f182..a53dd78c486 100644 --- a/src/coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml +++ b/src/coreComponents/mesh/docs/Ghosting/global_ghosting_setup.plantuml @@ -10,8 +10,6 @@ end box participant "..." as rtmpn participant "Rank n" as rn -==global numbering== - activate r0 #WhiteSmoke activate rtmp0 #WhiteSmoke activate rkm1 #WhiteSmoke @@ -37,7 +35,7 @@ r0 -[#tomato]->(90) rkm1++ #LimeGreen & rkm1 -[#red]>(40) rk & rkm1 -[#red]>(50) rkp1++ #LimeGreen & rkp1 -[#red]>(80) rk++ #LimeGreen -& rkp1 -[#red]>(70) rkm1glob +& rkp1 -[#red]>(70) rkm1 end group diff --git a/src/coreComponents/mesh/docs/Ghosting/test.plantuml b/src/coreComponents/mesh/docs/Ghosting/test.plantuml deleted file mode 100644 index fad8541bb08..00000000000 --- a/src/coreComponents/mesh/docs/Ghosting/test.plantuml +++ /dev/null @@ -1,5 +0,0 @@ -@startuml - -title Empty Diagram - -@enduml From 1f3c11265311f8b6a37667406113a5f63ecb8d67 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:45:26 -0700 Subject: [PATCH 009/106] add missing package --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index b1c5ba8da4b..fb65157537c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,6 +7,7 @@ build: os: "ubuntu-22.04" apt_packages: - curl + - graphviz - npm - openjdk-21-jre-headless - texlive From 7d25eef6356d7014b75726ccc2fccec31b71474d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:44:33 -0700 Subject: [PATCH 010/106] First version of the poc in python --- scripts/ghosting.py | 254 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 scripts/ghosting.py diff --git a/scripts/ghosting.py b/scripts/ghosting.py new file mode 100644 index 00000000000..f75cda8ae98 --- /dev/null +++ b/scripts/ghosting.py @@ -0,0 +1,254 @@ +from collections import defaultdict +from dataclasses import dataclass +from glob import glob +import logging +import sys +from typing import ( + Collection, + Dict, + Optional, + FrozenSet, + Mapping +) +from copy import deepcopy + +from vtkmodules.vtkCommonDataModel import ( + vtkUnstructuredGrid +) +from vtkmodules.vtkIOXML import ( + vtkXMLUnstructuredGridReader, +) +from vtkmodules.util.numpy_support import ( + vtk_to_numpy, + numpy_to_vtk, +) +from vtk import (vtkCell, + vtkBoundingBox) + +logger = logging.getLogger("ghosting") + + +def __read_vtu(vtk_input_file: str) -> Optional[vtkUnstructuredGrid]: + reader = vtkXMLUnstructuredGridReader() + logger.info(f"Testing file format \"{vtk_input_file}\" using XML format reader...") + if reader.CanReadFile(vtk_input_file): + reader.SetFileName(vtk_input_file) + logger.info(f"Reader matches. Reading file \"{vtk_input_file}\" using XML format reader.") + reader.Update() + return reader.GetOutput() + else: + logger.info("Reader did not match the input file format.") + return None + + +@dataclass(frozen=True) +class Node: + local: int # vtk index + global_: int # vtk global index + + +@dataclass(frozen=True) +class Edge: + nodes: tuple[int, int] # Local indices in the vtk mesh. Sorted. + + +@dataclass(frozen=True) +class MeshGraph: + nodes: Collection[Node] + edges: Collection[Edge] + + +def build_edges(mesh: vtkUnstructuredGrid, points_gids) -> Collection[Edge]: + tmp: set[Edge] = set() + for c in range(mesh.GetNumberOfCells()): + cell: vtkCell = mesh.GetCell(c) + for e in range(cell.GetNumberOfEdges()): + edge: vtkCell = cell.GetEdge(e) + ls = edge.GetPointId(0), edge.GetPointId(1) + # g0, g1 = points_gids[l0], points_gids[l1] + # tmp.add(tuple(sorted((g0, g1)))) + nodes = tuple(sorted(ls)) + tmp.add(Edge(nodes)) + return tuple(tmp) + + +def compute_graph(mesh: vtkUnstructuredGrid) -> MeshGraph: + num_points: int = mesh.GetNumberOfPoints() + points_gids = vtk_to_numpy(mesh.GetPointData().GetGlobalIds()) + nodes = [] + for l, g in enumerate(points_gids): + nodes.append(Node(l, g)) + edges: Collection[Edge] = build_edges(mesh, points_gids) + graph: MeshGraph = MeshGraph(tuple(nodes), edges) + return graph + + +def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Collection[Collection[int]]): + # [rank] -> [ [edge sorted global nodes] -> [edge index in the MeshGraph] ] + tmp: dict[int, dict[tuple[int, int], int]] = dict() + for rank, graph in enumerate(graphs): + d: dict[tuple[int, int], int] = {} + for ie, edge in enumerate(graph.edges): + gn0: int = graph.nodes[edge.nodes[0]].global_ + gn1: int = graph.nodes[edge.nodes[1]].global_ + gns: tuple[int, int] = tuple(sorted((gn0, gn1))) + d[gns] = ie + tmp[rank] = d + + # # Build a global to local map for the nodes + # ng2l: dict[int, tuple[int, int]] = {} # local is in fact (rank, local) + # for rank, graph in enumerate(graphs): + # for node in graph.nodes: + # ng2l[node.global_] = (rank, node.local) + + # raise ValueError("TODO") # Maybe `intersections` should be done neighborhood by neighborhood, so is `count`, as a tmp. + # [rank_a, rank_b] -> [Collection[Edge]] + # intersections: dict[tuple[int, ...], set[tuple[int, int]]] = defaultdict(set) + + rank_to_intersections: list[dict[tuple[int, ...], set[tuple[int, int]]]] = [] + for i in range(len(graphs)): + rank_to_intersections.append(defaultdict(set)) + + for current_rank, intersection in enumerate(rank_to_intersections): + count: dict[tuple[int, int], set[int]] = defaultdict(set) + other_ranks = neighbors[current_rank] + # for current_rank, other_ranks in enumerate(neighbors): + all_ranks = [current_rank, ] + other_ranks + for rank in all_ranks: + for edge in tmp[rank]: + count[edge].add(rank) + for edge, ranks in count.items(): + if len(ranks) > 0 and current_rank in ranks: + intersection[tuple(sorted(ranks))].add(edge) + + # Check if neighborhood is too wide... + for current_rank, other_ranks in enumerate(neighbors): + i = rank_to_intersections[current_rank] + useful_neighbors = set() + for ranks, edges in i.items(): + if edges: + useful_neighbors |= set(ranks) + print(f"Ranks '{set([current_rank, ] + other_ranks) - useful_neighbors}' are not required for rank {current_rank}.") + + # Now let's build the scan (or equivalent) + scans: list[Mapping[tuple[int, ...], int]] = [] + # for rank in range(len(graphs)): # PUT BACK + for rank, intersections in enumerate(rank_to_intersections): + scan = {} if len(scans) == 0 else deepcopy(scans[-1]) + for ranks, edges in intersections.items(): + # I want the current rank to have its own information for easy access to the offset, + # so there's no special case between what the local information and the information from the other ranks. + # Hence the `+ 1` so it's included in the `range` + if set(range(rank + 1)) & set(ranks): + scan[ranks] = len(edges) + scans.append(scan) + + offset_scans: list[Mapping[tuple[int, ...], int]] = [] + for scan in scans: + offset_scan: dict[tuple[int, ...], int] = dict() + i: int = 0 + for ranks in sorted(scan.keys()): + n: int = scan.get(ranks) + offset_scan[ranks] = i + i += n + offset_scans.append(offset_scan) + + # From `intersections`, let's mimic an intersection for the rank only + real_intersection: list[Mapping[tuple[int, ...], FrozenSet[tuple[int, int]]]] = [] + # for rank in range(len(graphs)): # PUT BACK + for rank, intersections in enumerate(rank_to_intersections): + i: Dict[tuple[int, ...], set[tuple[int, int]]] = dict() + for ranks, edges in intersections.items(): + if rank in ranks: + i[ranks] = edges + real_intersection.append(i) + + return real_intersection, offset_scans + + +def do_the_numbering(intersection: Mapping[tuple[int, ...], FrozenSet[tuple[int, int]]], + offset_scan: Mapping[tuple[int, ...], int]) -> Mapping[tuple[int, int], int]: + numbered_edges: dict[tuple[int, int], int] = dict() + for ranks, edges in sorted(intersection.items()): + off: int = offset_scan[ranks] + for edge in sorted(edges): + numbered_edges[edge] = off + off += 1 + return numbered_edges + + +def validation(numberings: Mapping[tuple[int, int], int]) -> int: + all_nodes = set() + all_edges = dict() + for numbering in numberings: + for nodes, gei in numbering.items(): + all_nodes |= set(nodes) + res = all_edges.get(nodes) + if res is None: + all_edges[nodes] = gei + elif res != nodes: + assert res == gei + expected_num_edges: int = 3 * 10 * 11 * 11 + expected_num_nodes: int = 11 * 11 * 11 + assert len(all_edges) == expected_num_edges + assert set(all_edges.values()) == set(range(expected_num_edges)) + assert all_nodes == set(range(expected_num_nodes)) + return 0 + + +def build_neighborhood(meshes: Collection[vtkUnstructuredGrid]) -> Collection[Collection[int]]: + bounding_boxes: list[vtkBoundingBox] = [] + for mesh in meshes: + bb = vtkBoundingBox() + bb.AddBox(mesh.GetBounds()) + bb.Inflate(0.1) + bounding_boxes.append(bb) + neighbors: list[list[int]] = [] + for rank, bb in enumerate(bounding_boxes): + n = [] + for other_rank, other_bb in enumerate(bounding_boxes): + if rank == other_rank: + continue + if bb.Intersects(other_bb): + n.append(other_rank) + neighbors.append(n) + return neighbors + + +def main() -> int: + # For each rank, contains the raw vtk mesh. + meshes: list[vtkUnstructuredGrid] = [] + pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" + # pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" + for file_name in glob(pattern): + m = __read_vtu(file_name) + if m: + meshes.append(m) + + # For each rank, contains the neighbor rank (candidates). + neighbors: Collection[Collection[int]] = build_neighborhood(meshes) + + # For each rank, contains the graph built upon the vtk mesh. + graphs = [] + for m in meshes: + graphs.append(compute_graph(m)) + + # Perform the core computation of the intersection. + # `intersections` will contain the intersection for each rank, + # while `offset_scans` will contains the offset for the global numbering, + # as it would be done in during the `MPI_scan`. + intersections: Collection[Mapping[tuple[int, ...], FrozenSet[tuple[int, int]]]] + offset_scans: Collection[Mapping[tuple[int, ...], int]] + intersections, offset_scans = find_overlapping_edges(graphs, neighbors) + + # Finish by doing the global numbering for real. + numberings: list[Mapping[tuple[int, int], int]] = [] + for i, o in zip(intersections, offset_scans): + numberings.append(do_the_numbering(i, o)) + + # Small validation + return validation(numberings) + + +if __name__ == '__main__': + sys.exit(main()) From 11f6331b2a121eb2ae577406afaefcf24fd5dca5 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:26:26 -0700 Subject: [PATCH 011/106] A little cleaning before sharing. --- scripts/ghosting.py | 125 +++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 72 deletions(-) diff --git a/scripts/ghosting.py b/scripts/ghosting.py index f75cda8ae98..a6dd4c3e717 100644 --- a/scripts/ghosting.py +++ b/scripts/ghosting.py @@ -1,14 +1,15 @@ from collections import defaultdict from dataclasses import dataclass from glob import glob +from itertools import chain import logging import sys from typing import ( Collection, - Dict, + Iterable, + Mapping, Optional, - FrozenSet, - Mapping + Sequence, ) from copy import deepcopy @@ -20,7 +21,6 @@ ) from vtkmodules.util.numpy_support import ( vtk_to_numpy, - numpy_to_vtk, ) from vtk import (vtkCell, vtkBoundingBox) @@ -65,15 +65,12 @@ def build_edges(mesh: vtkUnstructuredGrid, points_gids) -> Collection[Edge]: for e in range(cell.GetNumberOfEdges()): edge: vtkCell = cell.GetEdge(e) ls = edge.GetPointId(0), edge.GetPointId(1) - # g0, g1 = points_gids[l0], points_gids[l1] - # tmp.add(tuple(sorted((g0, g1)))) - nodes = tuple(sorted(ls)) + nodes: tuple[int, int] = tuple(sorted(ls)) tmp.add(Edge(nodes)) return tuple(tmp) def compute_graph(mesh: vtkUnstructuredGrid) -> MeshGraph: - num_points: int = mesh.GetNumberOfPoints() points_gids = vtk_to_numpy(mesh.GetPointData().GetGlobalIds()) nodes = [] for l, g in enumerate(points_gids): @@ -83,90 +80,75 @@ def compute_graph(mesh: vtkUnstructuredGrid) -> MeshGraph: return graph -def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Collection[Collection[int]]): - # [rank] -> [ [edge sorted global nodes] -> [edge index in the MeshGraph] ] - tmp: dict[int, dict[tuple[int, int], int]] = dict() +def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[tuple[int, int]]]]) -> Sequence[Mapping[tuple[int, ...], int]]: + scans: list[Mapping[tuple[int, ...], int]] = [] + for rank, intersections in enumerate(rank_to_intersections): + scan = {} if len(scans) == 0 else deepcopy(scans[-1]) + for ranks, edges in intersections.items(): + # I want the current rank to have its own information for easy access to the offset, + # so there's no special case between what the local information and the information from the other ranks. + # Hence the `+ 1` so it's included in the `range` + if set(range(rank + 1)) & set(ranks): + scan[ranks] = len(edges) + scans.append(scan) + + offset_scans: list[Mapping[tuple[int, ...], int]] = [] + for scan in scans: + offset_scan: dict[tuple[int, ...], int] = dict() + intersect: int = 0 + for ranks in sorted(scan.keys()): + n: int = scan.get(ranks) + offset_scan[ranks] = intersect + intersect += n + offset_scans.append(offset_scan) + + return offset_scans + + +def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Sequence[Collection[int]]): + # [rank] -> [edge sorted global nodes] + tmp: dict[int, set[tuple[int, int]]] = dict() for rank, graph in enumerate(graphs): - d: dict[tuple[int, int], int] = {} + # d: dict[tuple[int, int], int] = {} + d: set[tuple[int, int]] = set() for ie, edge in enumerate(graph.edges): gn0: int = graph.nodes[edge.nodes[0]].global_ gn1: int = graph.nodes[edge.nodes[1]].global_ gns: tuple[int, int] = tuple(sorted((gn0, gn1))) - d[gns] = ie + d.add(gns) tmp[rank] = d - # # Build a global to local map for the nodes - # ng2l: dict[int, tuple[int, int]] = {} # local is in fact (rank, local) - # for rank, graph in enumerate(graphs): - # for node in graph.nodes: - # ng2l[node.global_] = (rank, node.local) - - # raise ValueError("TODO") # Maybe `intersections` should be done neighborhood by neighborhood, so is `count`, as a tmp. - # [rank_a, rank_b] -> [Collection[Edge]] - # intersections: dict[tuple[int, ...], set[tuple[int, int]]] = defaultdict(set) - + # "Allocate"... rank_to_intersections: list[dict[tuple[int, ...], set[tuple[int, int]]]] = [] - for i in range(len(graphs)): + for intersect in range(len(graphs)): rank_to_intersections.append(defaultdict(set)) + # .... fill. for current_rank, intersection in enumerate(rank_to_intersections): count: dict[tuple[int, int], set[int]] = defaultdict(set) other_ranks = neighbors[current_rank] # for current_rank, other_ranks in enumerate(neighbors): - all_ranks = [current_rank, ] + other_ranks - for rank in all_ranks: + all_ranks_in_the_neighborhood = chain((current_rank,), other_ranks) + for rank in all_ranks_in_the_neighborhood: for edge in tmp[rank]: count[edge].add(rank) for edge, ranks in count.items(): if len(ranks) > 0 and current_rank in ranks: intersection[tuple(sorted(ranks))].add(edge) - # Check if neighborhood is too wide... + # For information, check if neighborhood is too wide... for current_rank, other_ranks in enumerate(neighbors): - i = rank_to_intersections[current_rank] - useful_neighbors = set() - for ranks, edges in i.items(): + intersect: dict[tuple[int, ...], set[tuple[int, int]]] = rank_to_intersections[current_rank] + useful_neighbors: set[tuple[int, ...]] = set() + for ranks, edges in intersect.items(): if edges: useful_neighbors |= set(ranks) - print(f"Ranks '{set([current_rank, ] + other_ranks) - useful_neighbors}' are not required for rank {current_rank}.") - - # Now let's build the scan (or equivalent) - scans: list[Mapping[tuple[int, ...], int]] = [] - # for rank in range(len(graphs)): # PUT BACK - for rank, intersections in enumerate(rank_to_intersections): - scan = {} if len(scans) == 0 else deepcopy(scans[-1]) - for ranks, edges in intersections.items(): - # I want the current rank to have its own information for easy access to the offset, - # so there's no special case between what the local information and the information from the other ranks. - # Hence the `+ 1` so it's included in the `range` - if set(range(rank + 1)) & set(ranks): - scan[ranks] = len(edges) - scans.append(scan) - - offset_scans: list[Mapping[tuple[int, ...], int]] = [] - for scan in scans: - offset_scan: dict[tuple[int, ...], int] = dict() - i: int = 0 - for ranks in sorted(scan.keys()): - n: int = scan.get(ranks) - offset_scan[ranks] = i - i += n - offset_scans.append(offset_scan) - - # From `intersections`, let's mimic an intersection for the rank only - real_intersection: list[Mapping[tuple[int, ...], FrozenSet[tuple[int, int]]]] = [] - # for rank in range(len(graphs)): # PUT BACK - for rank, intersections in enumerate(rank_to_intersections): - i: Dict[tuple[int, ...], set[tuple[int, int]]] = dict() - for ranks, edges in intersections.items(): - if rank in ranks: - i[ranks] = edges - real_intersection.append(i) + print(f"Ranks '{set([current_rank, ] + other_ranks) - useful_neighbors}' are not required by rank {current_rank}.") - return real_intersection, offset_scans + return rank_to_intersections -def do_the_numbering(intersection: Mapping[tuple[int, ...], FrozenSet[tuple[int, int]]], +def do_the_numbering(intersection: Mapping[tuple[int, ...], frozenset[tuple[int, int]]], offset_scan: Mapping[tuple[int, ...], int]) -> Mapping[tuple[int, int], int]: numbered_edges: dict[tuple[int, int], int] = dict() for ranks, edges in sorted(intersection.items()): @@ -177,7 +159,7 @@ def do_the_numbering(intersection: Mapping[tuple[int, ...], FrozenSet[tuple[int, return numbered_edges -def validation(numberings: Mapping[tuple[int, int], int]) -> int: +def validation(numberings: Iterable[Mapping[tuple[int, int], int]]) -> int: all_nodes = set() all_edges = dict() for numbering in numberings: @@ -218,8 +200,8 @@ def build_neighborhood(meshes: Collection[vtkUnstructuredGrid]) -> Collection[Co def main() -> int: # For each rank, contains the raw vtk mesh. meshes: list[vtkUnstructuredGrid] = [] - pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" - # pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" + # pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" + pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" for file_name in glob(pattern): m = __read_vtu(file_name) if m: @@ -237,9 +219,8 @@ def main() -> int: # `intersections` will contain the intersection for each rank, # while `offset_scans` will contains the offset for the global numbering, # as it would be done in during the `MPI_scan`. - intersections: Collection[Mapping[tuple[int, ...], FrozenSet[tuple[int, int]]]] - offset_scans: Collection[Mapping[tuple[int, ...], int]] - intersections, offset_scans = find_overlapping_edges(graphs, neighbors) + intersections: Collection[Mapping[tuple[int, ...], frozenset[tuple[int, int]]]] = find_overlapping_edges(graphs, neighbors) + offset_scans: Collection[Mapping[tuple[int, ...], int]] = mpi_scan(intersections) # Finish by doing the global numbering for real. numberings: list[Mapping[tuple[int, int], int]] = [] From 087897eea8960cb042e4700040562c407df6e641 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 4 Apr 2024 17:18:04 -0700 Subject: [PATCH 012/106] Little type hints improvements. --- scripts/ghosting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/ghosting.py b/scripts/ghosting.py index a6dd4c3e717..b59bf10efd0 100644 --- a/scripts/ghosting.py +++ b/scripts/ghosting.py @@ -58,7 +58,7 @@ class MeshGraph: edges: Collection[Edge] -def build_edges(mesh: vtkUnstructuredGrid, points_gids) -> Collection[Edge]: +def build_edges(mesh: vtkUnstructuredGrid) -> Collection[Edge]: tmp: set[Edge] = set() for c in range(mesh.GetNumberOfCells()): cell: vtkCell = mesh.GetCell(c) @@ -75,7 +75,7 @@ def compute_graph(mesh: vtkUnstructuredGrid) -> MeshGraph: nodes = [] for l, g in enumerate(points_gids): nodes.append(Node(l, g)) - edges: Collection[Edge] = build_edges(mesh, points_gids) + edges: Collection[Edge] = build_edges(mesh) graph: MeshGraph = MeshGraph(tuple(nodes), edges) return graph @@ -87,7 +87,7 @@ def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[ for ranks, edges in intersections.items(): # I want the current rank to have its own information for easy access to the offset, # so there's no special case between what the local information and the information from the other ranks. - # Hence the `+ 1` so it's included in the `range` + # Hence, the `+ 1` so it's included in the `range` if set(range(rank + 1)) & set(ranks): scan[ranks] = len(edges) scans.append(scan) @@ -105,7 +105,7 @@ def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[ return offset_scans -def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Sequence[Collection[int]]): +def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Sequence[Iterable[int]]): # [rank] -> [edge sorted global nodes] tmp: dict[int, set[tuple[int, int]]] = dict() for rank, graph in enumerate(graphs): @@ -178,7 +178,7 @@ def validation(numberings: Iterable[Mapping[tuple[int, int], int]]) -> int: return 0 -def build_neighborhood(meshes: Collection[vtkUnstructuredGrid]) -> Collection[Collection[int]]: +def build_neighborhood(meshes: Collection[vtkUnstructuredGrid]) -> Sequence[Sequence[int]]: bounding_boxes: list[vtkBoundingBox] = [] for mesh in meshes: bb = vtkBoundingBox() @@ -208,7 +208,7 @@ def main() -> int: meshes.append(m) # For each rank, contains the neighbor rank (candidates). - neighbors: Collection[Collection[int]] = build_neighborhood(meshes) + neighbors: Sequence[Sequence[int]] = build_neighborhood(meshes) # For each rank, contains the graph built upon the vtk mesh. graphs = [] @@ -217,7 +217,7 @@ def main() -> int: # Perform the core computation of the intersection. # `intersections` will contain the intersection for each rank, - # while `offset_scans` will contains the offset for the global numbering, + # while `offset_scans` will contain the offset for the global numbering, # as it would be done in during the `MPI_scan`. intersections: Collection[Mapping[tuple[int, ...], frozenset[tuple[int, int]]]] = find_overlapping_edges(graphs, neighbors) offset_scans: Collection[Mapping[tuple[int, ...], int]] = mpi_scan(intersections) From ae9a45641d3be1b29f29bb6ca3132f644071b5c7 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 5 Apr 2024 18:59:05 -0700 Subject: [PATCH 013/106] Small optimization --- scripts/ghosting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ghosting.py b/scripts/ghosting.py index b59bf10efd0..1c0a8cc30fc 100644 --- a/scripts/ghosting.py +++ b/scripts/ghosting.py @@ -93,13 +93,13 @@ def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[ scans.append(scan) offset_scans: list[Mapping[tuple[int, ...], int]] = [] - for scan in scans: + for rank, scan in enumerate(scans): offset_scan: dict[tuple[int, ...], int] = dict() intersect: int = 0 for ranks in sorted(scan.keys()): - n: int = scan.get(ranks) - offset_scan[ranks] = intersect - intersect += n + if max(ranks) >= rank: # Remove the entries that cannot play any more role. + offset_scan[ranks] = intersect + intersect += scan.get(ranks) offset_scans.append(offset_scan) return offset_scans From 778cd82a57553a2fa574b22174589c3f7c62c7de Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 5 Apr 2024 21:09:40 -0700 Subject: [PATCH 014/106] Revert "Small optimization" This reverts commit ae9a45641d3be1b29f29bb6ca3132f644071b5c7. --- scripts/ghosting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ghosting.py b/scripts/ghosting.py index 1c0a8cc30fc..b59bf10efd0 100644 --- a/scripts/ghosting.py +++ b/scripts/ghosting.py @@ -93,13 +93,13 @@ def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[ scans.append(scan) offset_scans: list[Mapping[tuple[int, ...], int]] = [] - for rank, scan in enumerate(scans): + for scan in scans: offset_scan: dict[tuple[int, ...], int] = dict() intersect: int = 0 for ranks in sorted(scan.keys()): - if max(ranks) >= rank: # Remove the entries that cannot play any more role. - offset_scan[ranks] = intersect - intersect += scan.get(ranks) + n: int = scan.get(ranks) + offset_scan[ranks] = intersect + intersect += n offset_scans.append(offset_scan) return offset_scans From 5d2bf531ae30104d1cc89e3ffcc5664ac4080377 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:02:58 -0700 Subject: [PATCH 015/106] First version that's exchanging edges properly. --- scripts/NamedType.py | 31 +++ scripts/ghosting.py | 6 +- src/coreComponents/common/GeosxConfig.hpp.in | 2 +- src/coreComponents/mesh/CMakeLists.txt | 21 ++ .../mesh/generators/NewGhosting.cpp | 247 ++++++++++++++++++ .../mesh/generators/NewGhosting.hpp | 31 +++ .../mesh/generators/VTKMeshGenerator.cpp | 11 + .../mesh/generators/VTKMeshGenerator.hpp | 4 + .../mesh/mpiCommunications/CommID.hpp | 2 +- 9 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 scripts/NamedType.py create mode 100644 src/coreComponents/mesh/generators/NewGhosting.cpp create mode 100644 src/coreComponents/mesh/generators/NewGhosting.hpp diff --git a/scripts/NamedType.py b/scripts/NamedType.py new file mode 100644 index 00000000000..b1d0173e95e --- /dev/null +++ b/scripts/NamedType.py @@ -0,0 +1,31 @@ +import logging + +import gdb + +logger = logging.getLogger("NamedType") + + +class NamedTypePrinter(object): + def __init__(self, val: gdb.Value): + self.val = val["value_"] + + def to_string(self) -> str: + return self.val + + +def build_array_printer(): + pp = gdb.printing.RegexpCollectionPrettyPrinter("NamedType") + pp.add_printer('fluent::NamedType', '^fluent::NamedType<.*>$', NamedTypePrinter) + return pp + + +try: + import gdb.printing + gdb.printing.register_pretty_printer(gdb.current_objfile(), build_array_printer()) +except ImportError: + logger.warning("Could not register LvArray pretty printers.") + + +# import debugpy +# debugpy.listen(("0.0.0.0", 64018)) +# debugpy.wait_for_client() diff --git a/scripts/ghosting.py b/scripts/ghosting.py index b59bf10efd0..b65ba557197 100644 --- a/scripts/ghosting.py +++ b/scripts/ghosting.py @@ -200,9 +200,9 @@ def build_neighborhood(meshes: Collection[vtkUnstructuredGrid]) -> Sequence[Sequ def main() -> int: # For each rank, contains the raw vtk mesh. meshes: list[vtkUnstructuredGrid] = [] - # pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" - pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" - for file_name in glob(pattern): + pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" + # pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" + for file_name in sorted(glob(pattern)): m = __read_vtu(file_name) if m: meshes.append(m) diff --git a/src/coreComponents/common/GeosxConfig.hpp.in b/src/coreComponents/common/GeosxConfig.hpp.in index 4b69fa93652..debea82b685 100644 --- a/src/coreComponents/common/GeosxConfig.hpp.in +++ b/src/coreComponents/common/GeosxConfig.hpp.in @@ -165,7 +165,7 @@ #cmakedefine petsc_VERSION @petsc_VERSION@ /// Version information for VTK -#cmakedefine VTK_VERSION @VTK_VERSION@ +// #cmakedefine VTK_VERSION @VTK_VERSION@ /// Version information for fmt #cmakedefine fmt_VERSION @fmt_VERSION@ diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index 1fafa2eb0a6..1ad47f2e12f 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -152,10 +152,30 @@ set( mesh_sources set( dependencyList ${parallelDeps} schema dataRepository constitutive finiteElement parmetis metis ) +include( FetchContent ) + +FetchContent_Declare( + NamedType + GIT_REPOSITORY https://github.com/joboccara/NamedType.git + GIT_TAG master +) + +FetchContent_GetProperties( NamedType ) +#FetchContent_MakeAvailable( namedtype ) +if( NOT namedtype_POPULATED ) + message( STATUS "Fetching NamedType..." ) + FetchContent_Populate( NamedType ) + add_subdirectory( ${namedtype_SOURCE_DIR} ${namedtype_BINARY_DIR} ) +endif() + +list( APPEND dependencyList NamedType ) + + if( ENABLE_VTK ) message(STATUS "Adding VTK readers") set( mesh_headers ${mesh_headers} generators/CollocatedNodes.hpp + generators/NewGhosting.hpp generators/VTKFaceBlockUtilities.hpp generators/VTKMeshGenerator.hpp generators/VTKMeshGeneratorTools.hpp @@ -164,6 +184,7 @@ if( ENABLE_VTK ) ) set( mesh_sources ${mesh_sources} generators/CollocatedNodes.cpp + generators/NewGhosting.cpp generators/VTKFaceBlockUtilities.cpp generators/VTKMeshGenerator.cpp generators/VTKMeshGeneratorTools.cpp diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp new file mode 100644 index 00000000000..a3107ed2402 --- /dev/null +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -0,0 +1,247 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "NewGhosting.hpp" + +#include "mesh/mpiCommunications/CommunicationTools.hpp" +#include "mesh/mpiCommunications/MPI_iCommData.hpp" +#include "mesh/mpiCommunications/NeighborCommunicator.hpp" + +#include "common/MpiWrapper.hpp" +#include "common/DataTypes.hpp" + +#include + +#include + +#include +#include + + +namespace geos::ghosting +{ + +using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; +using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable >; +using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; +using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable >; +using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; +using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; + +using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable >; + +//struct Edge +//{ +// std::pair< NodeGlbIdx, NodeGlbIdx > nodes; +//}; + +//struct Face +//{ +// std::vector< NodeGlbIdx > nodes; +//}; + +using Edge = std::pair< NodeGlbIdx, NodeGlbIdx >; +using Face = std::vector< NodeGlbIdx >; + +struct Exchange +{ + std::set< Edge > edges; +// std::set< Face > faces; +}; + +Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) +{ + vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); + + std::vector< Edge > tmp; + + // Pre-allocation of the temporary. + { + std::size_t numEdges = 0; + for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) + { + vtkCell * cell = mesh->GetCell( c ); + numEdges += cell->GetNumberOfEdges(); + } + tmp.reserve( numEdges ); + } + + // Filling the temporary. + for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) + { + vtkCell * cell = mesh->GetCell( c ); + for( auto e = 0; e < cell->GetNumberOfEdges(); ++e ) + { + vtkCell * edge = cell->GetEdge( e ); + vtkIdType const ln0 = edge->GetPointId( 0 ), ln1 = edge->GetPointId( 1 ); + vtkIdType const gn0 = globalPtIds->GetValue( ln0 ), gn1 = globalPtIds->GetValue( ln1 ); + tmp.emplace_back( std::minmax( { gn0, gn1 } ) ); + } + } + + // Removing the duplicates by copying into a `std::set`. + std::set< Edge > const edges{ tmp.cbegin(), tmp.cend() }; + + return { edges }; +} + +array1d< globalIndex > convertExchange( Exchange const & exchange ) +{ + array1d< globalIndex > result; + result.reserve( 2 * exchange.edges.size() ); + for( Edge const & edge: exchange.edges ) + { + result.emplace_back( edge.first.get() ); + result.emplace_back( edge.second.get() ); + } + + return result; +} + +Exchange convertExchange( array1d< globalIndex > const & input ) +{ + Exchange exchange; + + for( auto i = 0; i < input.size(); ++ ++i ) + { + NodeGlbIdx gn0{ input[i] }, gn1{ input[i + 1] }; + exchange.edges.insert( std::minmax( gn0, gn1 ) ); + } + + return exchange; +} + +/** + * @brief + * @param exchanged + * @param neighborhood including current rank + */ +std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map< MpiRank, Exchange > const & exchanged, + MpiRank curRank, + std::set< MpiRank > const & neighborhood ) +{ + GEOS_LOG_RANK( "Starting findOverlappingEdges" ); + GEOS_LOG_RANK( "exchanged.size() = " << exchanged.size() ); + + std::map< Edge, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? + for( MpiRank const & rank: neighborhood ) + { + for( Edge const & edge: exchanged.at( rank ).edges ) + { + counts[edge].insert( rank ); + } + } + + std::map< std::set< MpiRank >, std::set< Edge > > ranksToIntersections; + for( auto const & [edge, ranks]: counts ) + { + if( ranks.find( curRank ) != ranks.cend() ) + { + ranksToIntersections[ranks].insert( edge ); + } + } + + // Checking if neighborhood is too wide... // TODO do we care? + std::set< MpiRank > usefulNeighbors; + for( auto const & [ranks, edges]: ranksToIntersections ) + { + if( not edges.empty() ) + { + usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); + } + } + std::vector< MpiRank > uselessNeighbors; + std::set_difference( neighborhood.cbegin(), neighborhood.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); + // TODO... Remove the neighbors? + GEOS_LOG_RANK( "Ending findOverlappingEdges" ); + + return ranksToIntersections; +} + +// TODO Duplicated +std::map< MpiRank, Exchange > exchange( int commId, + std::vector< NeighborCommunicator > & neighbors, + Exchange && data ) +{ + MPI_iCommData commData( commId ); + integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); + commData.resize( numNeighbors ); + + + array1d< globalIndex > const cv = convertExchange( data ); + + for( integer i = 0; i < numNeighbors; ++i ) + { + neighbors[i].mpiISendReceiveSizes( cv, + commData.mpiSendBufferSizeRequest( i ), + commData.mpiRecvBufferSizeRequest( i ), + commId, + MPI_COMM_GEOSX ); + } + + MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus() ); + MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus() ); + + array1d< array1d< globalIndex > > tmpOutput( neighbors.size() ); + + for( integer i = 0; i < numNeighbors; ++i ) + { + neighbors[i].mpiISendReceiveData( cv, + commData.mpiSendBufferRequest( i ), + tmpOutput[i], + commData.mpiRecvBufferRequest( i ), + commId, + MPI_COMM_GEOSX ); + } + MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferRequest(), commData.mpiSendBufferStatus() ); + MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferRequest(), commData.mpiRecvBufferStatus() ); + + std::map< MpiRank, Exchange > output; + for( auto i = 0; i < numNeighbors; ++i ) + { + output[MpiRank{ neighbors[i].neighborRank() }] = convertExchange( tmpOutput[i] ); + } + output[MpiRank{ MpiWrapper::commRank() }] = std::move( data ); + return output; +} + + +void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors ) +{ + // Now we exchange the data with our neighbors. + MpiRank const curRank{ MpiWrapper::commRank() }; + + std::vector< NeighborCommunicator > ncs; + for( int const & rank: neighbors ) + { + ncs.emplace_back( rank ); + } + + std::set< MpiRank > neighborhood; + for( int const & rank: neighbors ) + { + neighborhood.insert( MpiRank{ rank } ); + } + neighborhood.insert( curRank ); + + CommID const commId = CommunicationTools::getInstance().getCommID(); + std::map< MpiRank, Exchange > const exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); + + std::map< std::set< MpiRank >, std::set< Edge > > const overlappingEdges = findOverlappingEdges( exchanged, curRank, neighborhood ); +} + +} // end of namespace geos::ghosting diff --git a/src/coreComponents/mesh/generators/NewGhosting.hpp b/src/coreComponents/mesh/generators/NewGhosting.hpp new file mode 100644 index 00000000000..b908d9cfb2c --- /dev/null +++ b/src/coreComponents/mesh/generators/NewGhosting.hpp @@ -0,0 +1,31 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_NEWGHOSTING_HPP +#define GEOS_NEWGHOSTING_HPP + +#include +#include + +#include + +namespace geos::ghosting +{ + +void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors ); + +} // end of namespace ghosting + +#endif // include guard diff --git a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp index b45ed3f9e30..72ebf48c0d1 100644 --- a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp +++ b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp @@ -21,6 +21,7 @@ #include "mesh/generators/VTKFaceBlockUtilities.hpp" #include "mesh/generators/VTKMeshGeneratorTools.hpp" #include "mesh/generators/CellBlockManager.hpp" +#include "mesh/generators/NewGhosting.hpp" #include "common/DataTypes.hpp" #include @@ -74,6 +75,11 @@ VTKMeshGenerator::VTKMeshGenerator( string const & name, " If set to 0 (default value), the GlobalId arrays in the input mesh are used if available, and generated otherwise." " If set to a negative value, the GlobalId arrays in the input mesh are not used, and generated global Ids are automatically generated." " If set to a positive value, the GlobalId arrays in the input mesh are used and required, and the simulation aborts if they are not available" ); + + registerWrapper( viewKeyStruct::useNewGhostingString(), &m_useNewGhosting ). + setInputFlag( InputFlags::OPTIONAL ). + setApplyDefaultValue( 0 ). + setDescription( "Controls the use of the new ghosting implementation." ); } void VTKMeshGenerator::fillCellBlockManager( CellBlockManager & cellBlockManager, SpatialPartition & partition ) @@ -102,6 +108,11 @@ void VTKMeshGenerator::fillCellBlockManager( CellBlockManager & cellBlockManager } GEOS_LOG_RANK_0( GEOS_FMT( "{} '{}': generating GEOSX mesh data structure", catalogName(), getName() ) ); + if( m_useNewGhosting ) + { + GEOS_LOG_RANK( "Here we go, new ghosting!" ); + ghosting::doTheNewGhosting( m_vtkMesh, partition.getMetisNeighborList() ); + } GEOS_LOG_LEVEL_RANK_0( 2, " preprocessing..." ); m_cellMap = vtk::buildCellMap( *m_vtkMesh, m_attributeName ); diff --git a/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp b/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp index 3fae5d6d343..83845f27c42 100644 --- a/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp +++ b/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp @@ -109,6 +109,7 @@ class VTKMeshGenerator : public ExternalMeshGeneratorBase constexpr static char const * partitionRefinementString() { return "partitionRefinement"; } constexpr static char const * partitionMethodString() { return "partitionMethod"; } constexpr static char const * useGlobalIdsString() { return "useGlobalIds"; } + constexpr static char const * useNewGhostingString() { return "useNewGhosting"; } }; /// @endcond @@ -149,6 +150,9 @@ class VTKMeshGenerator : public ExternalMeshGeneratorBase /// Whether global id arrays should be used, if available integer m_useGlobalIds = 0; + // Whether we should use the new ghosting implementation. + integer m_useNewGhosting = 0; + /// Method (library) used to partition the mesh vtk::PartitionMethod m_partitionMethod = vtk::PartitionMethod::parmetis; diff --git a/src/coreComponents/mesh/mpiCommunications/CommID.hpp b/src/coreComponents/mesh/mpiCommunications/CommID.hpp index 143e55eb18c..7c904234f07 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommID.hpp +++ b/src/coreComponents/mesh/mpiCommunications/CommID.hpp @@ -59,7 +59,7 @@ class CommID CommID & operator=( CommID && ) = delete; /// user defined conversion operator to int - constexpr operator int() + constexpr operator int() const { return m_id; } private: From bcd7fba87385f4987c9b9bea105ff44faf0f90f6 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 11 Apr 2024 23:34:34 -0700 Subject: [PATCH 016/106] Use hollow meshes for neighboring ranks (optimization) --- scripts/ghosting.py | 79 +++++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/scripts/ghosting.py b/scripts/ghosting.py index b65ba557197..38640c71614 100644 --- a/scripts/ghosting.py +++ b/scripts/ghosting.py @@ -6,6 +6,7 @@ import sys from typing import ( Collection, + Generator, Iterable, Mapping, Optional, @@ -14,16 +15,22 @@ from copy import deepcopy from vtkmodules.vtkCommonDataModel import ( - vtkUnstructuredGrid + vtkBoundingBox, + vtkCell, + vtkPolyData, + vtkUnstructuredGrid, ) from vtkmodules.vtkIOXML import ( vtkXMLUnstructuredGridReader, ) +from vtkmodules.vtkFiltersGeometry import ( + vtkDataSetSurfaceFilter, +) from vtkmodules.util.numpy_support import ( vtk_to_numpy, ) -from vtk import (vtkCell, - vtkBoundingBox) +# from vtk import (vtkCell, +# vtkBoundingBox) logger = logging.getLogger("ghosting") @@ -58,9 +65,11 @@ class MeshGraph: edges: Collection[Edge] -def build_edges(mesh: vtkUnstructuredGrid) -> Collection[Edge]: +def build_edges(mesh: vtkUnstructuredGrid, cells: Generator[int, None, None] | None) -> Collection[Edge]: + if cells is None: + cells = range(mesh.GetNumberOfCells()) tmp: set[Edge] = set() - for c in range(mesh.GetNumberOfCells()): + for c in cells: cell: vtkCell = mesh.GetCell(c) for e in range(cell.GetNumberOfEdges()): edge: vtkCell = cell.GetEdge(e) @@ -70,16 +79,32 @@ def build_edges(mesh: vtkUnstructuredGrid) -> Collection[Edge]: return tuple(tmp) -def compute_graph(mesh: vtkUnstructuredGrid) -> MeshGraph: +def compute_graph(mesh: vtkUnstructuredGrid, cells=None) -> MeshGraph: points_gids = vtk_to_numpy(mesh.GetPointData().GetGlobalIds()) nodes = [] for l, g in enumerate(points_gids): nodes.append(Node(l, g)) - edges: Collection[Edge] = build_edges(mesh) + edges: Collection[Edge] = build_edges(mesh, cells) graph: MeshGraph = MeshGraph(tuple(nodes), edges) return graph +def compute_hollow_graph(mesh: vtkUnstructuredGrid) -> MeshGraph: + f = vtkDataSetSurfaceFilter() + f.PassThroughCellIdsOn() + f.PassThroughPointIdsOff() + f.FastModeOff() + + # Note that we do not need the original points, but we could keep them as well if needed + original_cells_key = "ORIGINAL_CELLS" + f.SetOriginalCellIdsName(original_cells_key) + + boundary_mesh = vtkPolyData() + f.UnstructuredGridExecute(mesh, boundary_mesh) + original_cells = vtk_to_numpy(boundary_mesh.GetCellData().GetArray(original_cells_key)) + return compute_graph(mesh, set(original_cells)) + + def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[tuple[int, int]]]]) -> Sequence[Mapping[tuple[int, ...], int]]: scans: list[Mapping[tuple[int, ...], int]] = [] for rank, intersections in enumerate(rank_to_intersections): @@ -105,8 +130,10 @@ def mpi_scan(rank_to_intersections: Iterable[Mapping[tuple[int, ...], frozenset[ return offset_scans -def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Sequence[Iterable[int]]): - # [rank] -> [edge sorted global nodes] +def build_rank_to_edges(graphs: Iterable[MeshGraph]) -> dict[int, set[tuple[int, int]]]: + """ + Builds the mapping from ranks to the edges (as pairs of global node indices) + """ tmp: dict[int, set[tuple[int, int]]] = dict() for rank, graph in enumerate(graphs): # d: dict[tuple[int, int], int] = {} @@ -117,23 +144,31 @@ def find_overlapping_edges(graphs: Collection[MeshGraph], neighbors: Sequence[It gns: tuple[int, int] = tuple(sorted((gn0, gn1))) d.add(gns) tmp[rank] = d + return tmp + + +def find_overlapping_edges(graphs: Collection[MeshGraph], + hollow_graphs: Collection[MeshGraph], + neighbors: Sequence[Iterable[int]]): + # [rank] -> [edge sorted global nodes] + tmp_g: Mapping[int, set[tuple[int, int]]] = build_rank_to_edges(graphs) + tmp_hg: Mapping[int, set[tuple[int, int]]] = build_rank_to_edges(hollow_graphs) - # "Allocate"... + # "Instanciate"... rank_to_intersections: list[dict[tuple[int, ...], set[tuple[int, int]]]] = [] for intersect in range(len(graphs)): rank_to_intersections.append(defaultdict(set)) - # .... fill. + # ... fill. for current_rank, intersection in enumerate(rank_to_intersections): count: dict[tuple[int, int], set[int]] = defaultdict(set) - other_ranks = neighbors[current_rank] - # for current_rank, other_ranks in enumerate(neighbors): - all_ranks_in_the_neighborhood = chain((current_rank,), other_ranks) - for rank in all_ranks_in_the_neighborhood: - for edge in tmp[rank]: + for edge in tmp_g[current_rank]: + count[edge].add(current_rank) + for rank in neighbors[current_rank]: + for edge in tmp_hg[rank]: count[edge].add(rank) for edge, ranks in count.items(): - if len(ranks) > 0 and current_rank in ranks: + if current_rank in ranks: intersection[tuple(sorted(ranks))].add(edge) # For information, check if neighborhood is too wide... @@ -200,8 +235,8 @@ def build_neighborhood(meshes: Collection[vtkUnstructuredGrid]) -> Sequence[Sequ def main() -> int: # For each rank, contains the raw vtk mesh. meshes: list[vtkUnstructuredGrid] = [] - pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" - # pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" + # pattern: str = "/Users/j0436735/Downloads/meshes-cube/main-*.vtu" + pattern: str = "/Users/j0436735/Downloads/meshes-cube-25/main-*.vtu" for file_name in sorted(glob(pattern)): m = __read_vtu(file_name) if m: @@ -211,15 +246,17 @@ def main() -> int: neighbors: Sequence[Sequence[int]] = build_neighborhood(meshes) # For each rank, contains the graph built upon the vtk mesh. - graphs = [] + graphs: list[MeshGraph] = [] + hollow_graphs: list[MeshGraph] = [] for m in meshes: graphs.append(compute_graph(m)) + hollow_graphs.append(compute_hollow_graph(m)) # Perform the core computation of the intersection. # `intersections` will contain the intersection for each rank, # while `offset_scans` will contain the offset for the global numbering, # as it would be done in during the `MPI_scan`. - intersections: Collection[Mapping[tuple[int, ...], frozenset[tuple[int, int]]]] = find_overlapping_edges(graphs, neighbors) + intersections: Collection[Mapping[tuple[int, ...], frozenset[tuple[int, int]]]] = find_overlapping_edges(graphs, hollow_graphs, neighbors) offset_scans: Collection[Mapping[tuple[int, ...], int]] = mpi_scan(intersections) # Finish by doing the global numbering for real. From 1de9f067df685f59fd5d21b150fedad226d39dbe Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 12 Apr 2024 08:55:51 -0700 Subject: [PATCH 017/106] Start work for Faces --- .../mesh/generators/NewGhosting.cpp | 90 ++++++++++++++----- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index a3107ed2402..f72da49a7a3 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -54,48 +54,84 @@ using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, f //}; using Edge = std::pair< NodeGlbIdx, NodeGlbIdx >; -using Face = std::vector< NodeGlbIdx >; +using Face = std::set< NodeGlbIdx >; struct Exchange { std::set< Edge > edges; -// std::set< Face > faces; + std::set< Face > faces; +// SortedArray< Edge > edges; +// SortedArray< Face > faces; }; +template< bool DO_PACKING > +localIndex +Pack( buffer_unit_type *& buffer, + Exchange const & exchange ) +{ + localIndex sizeOfPackedChars = 0; + sizeOfPackedChars += Pack< DO_PACKING >( buffer, exchange.edges ); + sizeOfPackedChars += Pack< DO_PACKING >( buffer, exchange.faces ); + return sizeOfPackedChars; +} + + Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) { vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); - std::vector< Edge > tmp; + std::vector< Edge > tmpEdges; + std::vector< Face > tmpFaces; - // Pre-allocation of the temporary. + // Pre-allocation of the temporary vectors. { std::size_t numEdges = 0; + std::size_t numFaces = 0; for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) { vtkCell * cell = mesh->GetCell( c ); numEdges += cell->GetNumberOfEdges(); + numFaces += cell->GetNumberOfFaces(); } - tmp.reserve( numEdges ); + tmpEdges.reserve( numEdges ); + tmpFaces.reserve( numFaces ); } // Filling the temporary. for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) { vtkCell * cell = mesh->GetCell( c ); + for( auto e = 0; e < cell->GetNumberOfEdges(); ++e ) { vtkCell * edge = cell->GetEdge( e ); - vtkIdType const ln0 = edge->GetPointId( 0 ), ln1 = edge->GetPointId( 1 ); - vtkIdType const gn0 = globalPtIds->GetValue( ln0 ), gn1 = globalPtIds->GetValue( ln1 ); - tmp.emplace_back( std::minmax( { gn0, gn1 } ) ); + vtkIdType const ln0 = edge->GetPointId( 0 ); + vtkIdType const ln1 = edge->GetPointId( 1 ); + vtkIdType const gn0 = globalPtIds->GetValue( ln0 ); + vtkIdType const gn1 = globalPtIds->GetValue( ln1 ); + tmpEdges.emplace_back( std::minmax( { gn0, gn1 } ) ); + } + + for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) + { + vtkCell * face = cell->GetFace( f ); + vtkIdList * pids = face->GetPointIds(); + Face ff; + for( auto i = 0; i < pids->GetNumberOfIds(); ++i ) + { + vtkIdType const lni = face->GetPointId( i ); + vtkIdType const gni = globalPtIds->GetValue( lni ); + ff.insert( NodeGlbIdx{ gni } ); + } + tmpFaces.emplace_back( ff ); } } // Removing the duplicates by copying into a `std::set`. - std::set< Edge > const edges{ tmp.cbegin(), tmp.cend() }; + std::set< Edge > edges{ tmpEdges.cbegin(), tmpEdges.cend() }; // SortedArray requires the input to be sorted already. + std::set< Face > faces{ tmpFaces.cbegin(), tmpFaces.cend() }; - return { edges }; + return { std::move( edges ), std::move( faces ) }; } array1d< globalIndex > convertExchange( Exchange const & exchange ) @@ -127,21 +163,33 @@ Exchange convertExchange( array1d< globalIndex > const & input ) /** * @brief * @param exchanged - * @param neighborhood including current rank + * @param neighbors excluding current rank */ std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map< MpiRank, Exchange > const & exchanged, MpiRank curRank, - std::set< MpiRank > const & neighborhood ) + std::set< MpiRank > const & neighbors ) { GEOS_LOG_RANK( "Starting findOverlappingEdges" ); GEOS_LOG_RANK( "exchanged.size() = " << exchanged.size() ); std::map< Edge, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? - for( MpiRank const & rank: neighborhood ) + // We "register" all the edges of the current rank: they are the only one we're interested in. + for( Edge const & edge: exchanged.at( curRank ).edges ) + { + counts.emplace_hint( counts.end(), edge, std::set< MpiRank >{ curRank } ); + } + + // We now loop on the neighbor edges. + // If a neighbor has an edge in common with the current rank, they we store it. + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. { - for( Edge const & edge: exchanged.at( rank ).edges ) + for( Edge const & edge: exchanged.at( neighborRank ).edges ) { - counts[edge].insert( rank ); + auto it = counts.find( edge ); + if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. + { + it->second.insert( neighborRank ); + } } } @@ -154,7 +202,7 @@ std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map } } - // Checking if neighborhood is too wide... // TODO do we care? + // Checking if neighbors is too wide... // TODO do we care? std::set< MpiRank > usefulNeighbors; for( auto const & [ranks, edges]: ranksToIntersections ) { @@ -164,7 +212,7 @@ std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map } } std::vector< MpiRank > uselessNeighbors; - std::set_difference( neighborhood.cbegin(), neighborhood.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); + std::set_difference( neighbors.cbegin(), neighbors.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); // TODO... Remove the neighbors? GEOS_LOG_RANK( "Ending findOverlappingEdges" ); @@ -218,6 +266,7 @@ std::map< MpiRank, Exchange > exchange( int commId, return output; } +using ScannedOffsets = std::map< std::set< MpiRank >, integer >; void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< int > const & neighbors ) @@ -231,17 +280,16 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, ncs.emplace_back( rank ); } - std::set< MpiRank > neighborhood; + std::set< MpiRank > neighbors_; for( int const & rank: neighbors ) { - neighborhood.insert( MpiRank{ rank } ); + neighbors_.insert( MpiRank{ rank } ); } - neighborhood.insert( curRank ); CommID const commId = CommunicationTools::getInstance().getCommID(); std::map< MpiRank, Exchange > const exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); - std::map< std::set< MpiRank >, std::set< Edge > > const overlappingEdges = findOverlappingEdges( exchanged, curRank, neighborhood ); + std::map< std::set< MpiRank >, std::set< Edge > > const overlappingEdges = findOverlappingEdges( exchanged, curRank, neighbors_ ); } } // end of namespace geos::ghosting From 5b5d14074ae236c5dc99a8f488aa4a69e2fa9acb Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:02:51 -0700 Subject: [PATCH 018/106] Exchange hollow information instead. --- .../mesh/generators/NewGhosting.cpp | 80 +++++++++++++++---- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index f72da49a7a3..d709dbd5260 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -23,7 +23,10 @@ #include +#include #include +#include +#include #include #include @@ -76,7 +79,30 @@ Pack( buffer_unit_type *& buffer, } -Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) +std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) +{ + auto f = vtkDataSetSurfaceFilter::New(); + f->PassThroughCellIdsOn(); + f->PassThroughPointIdsOff(); + f->FastModeOff(); + + string const originalCellsKey = "ORIGINAL_CELLS"; + f->SetOriginalCellIdsName( originalCellsKey.c_str() ); + auto boundaryMesh = vtkPolyData::New(); + f->UnstructuredGridExecute( mesh, boundaryMesh ); + vtkIdTypeArray const * originalCells = vtkIdTypeArray::FastDownCast( boundaryMesh->GetCellData()->GetArray( originalCellsKey.c_str() ) ); + + std::set< vtkIdType > boundaryCellIdxs; + for( auto i = 0; i < originalCells->GetNumberOfTuples(); ++i ) + { + boundaryCellIdxs.insert( originalCells->GetValue( i ) ); + } + + return boundaryCellIdxs; +} + + +Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdType > const & cellIds ) { vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); @@ -87,7 +113,7 @@ Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) { std::size_t numEdges = 0; std::size_t numFaces = 0; - for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) + for( vtkIdType const & c : cellIds ) { vtkCell * cell = mesh->GetCell( c ); numEdges += cell->GetNumberOfEdges(); @@ -134,6 +160,7 @@ Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) return { std::move( edges ), std::move( faces ) }; } + array1d< globalIndex > convertExchange( Exchange const & exchange ) { array1d< globalIndex > result; @@ -147,6 +174,24 @@ array1d< globalIndex > convertExchange( Exchange const & exchange ) return result; } + +Exchange buildFullData( vtkSmartPointer< vtkDataSet > mesh ) +{ + std::set< vtkIdType > cellIds; + for( vtkIdType i = 0; i < mesh->GetNumberOfCells(); ++i ) + { + cellIds.insert( cellIds.end(), i ); + } + + return buildSpecificData( mesh, cellIds ); +} + + +Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) +{ + return buildSpecificData( mesh, extractBoundaryCells( mesh ) ); +} + Exchange convertExchange( array1d< globalIndex > const & input ) { Exchange exchange; @@ -222,7 +267,7 @@ std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map // TODO Duplicated std::map< MpiRank, Exchange > exchange( int commId, std::vector< NeighborCommunicator > & neighbors, - Exchange && data ) + Exchange const & data ) { MPI_iCommData commData( commId ); integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); @@ -243,13 +288,13 @@ std::map< MpiRank, Exchange > exchange( int commId, MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus() ); MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus() ); - array1d< array1d< globalIndex > > tmpOutput( neighbors.size() ); + array1d< array1d< globalIndex > > rawExchanged( neighbors.size() ); for( integer i = 0; i < numNeighbors; ++i ) { neighbors[i].mpiISendReceiveData( cv, commData.mpiSendBufferRequest( i ), - tmpOutput[i], + rawExchanged[i], commData.mpiRecvBufferRequest( i ), commId, MPI_COMM_GEOSX ); @@ -260,36 +305,43 @@ std::map< MpiRank, Exchange > exchange( int commId, std::map< MpiRank, Exchange > output; for( auto i = 0; i < numNeighbors; ++i ) { - output[MpiRank{ neighbors[i].neighborRank() }] = convertExchange( tmpOutput[i] ); + output[MpiRank{ neighbors[i].neighborRank() }] = convertExchange( rawExchanged[i] ); } - output[MpiRank{ MpiWrapper::commRank() }] = std::move( data ); return output; } using ScannedOffsets = std::map< std::set< MpiRank >, integer >; + void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< int > const & neighbors ) + std::set< MpiRank > const & neighbors ) { // Now we exchange the data with our neighbors. MpiRank const curRank{ MpiWrapper::commRank() }; std::vector< NeighborCommunicator > ncs; - for( int const & rank: neighbors ) + for( MpiRank const & rank: neighbors ) { - ncs.emplace_back( rank ); + ncs.emplace_back( rank.get() ); } + CommID const commId = CommunicationTools::getInstance().getCommID(); + std::map< MpiRank, Exchange > exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); + exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); + + std::map< std::set< MpiRank >, std::set< Edge > > const overlappingEdges = findOverlappingEdges( exchanged, curRank, neighbors ); +} + +void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors ) +{ std::set< MpiRank > neighbors_; for( int const & rank: neighbors ) { neighbors_.insert( MpiRank{ rank } ); } - CommID const commId = CommunicationTools::getInstance().getCommID(); - std::map< MpiRank, Exchange > const exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); - - std::map< std::set< MpiRank >, std::set< Edge > > const overlappingEdges = findOverlappingEdges( exchanged, curRank, neighbors_ ); + return doTheNewGhosting(mesh, neighbors_); } } // end of namespace geos::ghosting From aac9c2f70250ddf49d073214aa6eca9c7e72dbb8 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:27:35 -0700 Subject: [PATCH 019/106] Exchanging faces as well. --- .../mesh/generators/NewGhosting.cpp | 110 +++++++++++++----- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index d709dbd5260..f376fbca27a 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -102,7 +102,8 @@ std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) } -Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdType > const & cellIds ) +Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, + std::set< vtkIdType > const & cellIds ) { vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); @@ -113,7 +114,7 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdT { std::size_t numEdges = 0; std::size_t numFaces = 0; - for( vtkIdType const & c : cellIds ) + for( vtkIdType const & c: cellIds ) { vtkCell * cell = mesh->GetCell( c ); numEdges += cell->GetNumberOfEdges(); @@ -163,18 +164,67 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdT array1d< globalIndex > convertExchange( Exchange const & exchange ) { + std::size_t const edgeSize = 1 + 2 * std::size( exchange.edges ); + std::size_t faceSize = 1; + for( Face const & face: exchange.faces ) + { + faceSize += 1 + std::size( face ); // `+1` because we need to store the size so we know where to stop. + } + array1d< globalIndex > result; - result.reserve( 2 * exchange.edges.size() ); + result.reserve( edgeSize + faceSize ); + + result.emplace_back( std::size( exchange.edges ) ); for( Edge const & edge: exchange.edges ) { result.emplace_back( edge.first.get() ); result.emplace_back( edge.second.get() ); } + result.emplace_back( std::size( exchange.faces ) ); + for( Face const & face: exchange.faces ) + { + result.emplace_back( std::size( face ) ); + for( NodeGlbIdx const & n: face ) + { + result.emplace_back( n.get() ); + } + } return result; } +Exchange convertExchange( array1d< globalIndex > const & input ) +{ + Exchange exchange; + + globalIndex const numEdges = input[0]; + int i; + for( i = 1; i < 2 * numEdges + 1; i += 2 ) + { + NodeGlbIdx gn0{ input[i] }, gn1{ input[i + 1] }; + exchange.edges.insert( std::minmax( gn0, gn1 ) ); + } + GEOS_ASSERT_EQ( std::size_t(numEdges), exchange.edges.size() ); + + globalIndex const numFaces = input[i]; + for( ++i; i < input.size(); ) + { + auto const s = input[i++]; + std::set< NodeGlbIdx > face; + for( int j = 0; j < s; ++j, ++i ) + { + face.insert( NodeGlbIdx{ input[i] } ); + } + exchange.faces.insert( face ); + } + + GEOS_ASSERT_EQ( std::size_t(numFaces), exchange.faces.size() ); + + return exchange; +} + + Exchange buildFullData( vtkSmartPointer< vtkDataSet > mesh ) { std::set< vtkIdType > cellIds; @@ -192,31 +242,22 @@ Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) return buildSpecificData( mesh, extractBoundaryCells( mesh ) ); } -Exchange convertExchange( array1d< globalIndex > const & input ) +struct Buckets { - Exchange exchange; + std::map< std::set< MpiRank >, std::set< Edge > > edges; + std::map< std::set< MpiRank >, std::set< Face > > faces; +}; - for( auto i = 0; i < input.size(); ++ ++i ) - { - NodeGlbIdx gn0{ input[i] }, gn1{ input[i + 1] }; - exchange.edges.insert( std::minmax( gn0, gn1 ) ); - } - - return exchange; -} /** * @brief * @param exchanged * @param neighbors excluding current rank */ -std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map< MpiRank, Exchange > const & exchanged, - MpiRank curRank, - std::set< MpiRank > const & neighbors ) +Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, + MpiRank curRank, + std::set< MpiRank > const & neighbors ) { - GEOS_LOG_RANK( "Starting findOverlappingEdges" ); - GEOS_LOG_RANK( "exchanged.size() = " << exchanged.size() ); - std::map< Edge, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? // We "register" all the edges of the current rank: they are the only one we're interested in. for( Edge const & edge: exchanged.at( curRank ).edges ) @@ -238,18 +279,34 @@ std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map } } - std::map< std::set< MpiRank >, std::set< Edge > > ranksToIntersections; + std::map< std::set< MpiRank >, std::set< Edge > > edgeBuckets; for( auto const & [edge, ranks]: counts ) { if( ranks.find( curRank ) != ranks.cend() ) { - ranksToIntersections[ranks].insert( edge ); + edgeBuckets[ranks].insert( edge ); } } + std::map< std::set< MpiRank >, std::set< Face > > faceBuckets; + std::set< Face > curFaces = exchanged.at( curRank ).faces; + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + { + for( Face const & face: exchanged.at( neighborRank ).faces ) + { + auto it = curFaces.find( face ); + if( it != curFaces.cend() ) + { + faceBuckets[{ curRank, neighborRank }].insert( *it ); + curFaces.erase( it ); + } + } + } + faceBuckets[{ curRank }] = curFaces; + // Checking if neighbors is too wide... // TODO do we care? std::set< MpiRank > usefulNeighbors; - for( auto const & [ranks, edges]: ranksToIntersections ) + for( auto const & [ranks, edges]: edgeBuckets ) { if( not edges.empty() ) { @@ -259,11 +316,11 @@ std::map< std::set< MpiRank >, std::set< Edge > > findOverlappingEdges( std::map std::vector< MpiRank > uselessNeighbors; std::set_difference( neighbors.cbegin(), neighbors.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); // TODO... Remove the neighbors? - GEOS_LOG_RANK( "Ending findOverlappingEdges" ); - return ranksToIntersections; + return { edgeBuckets, faceBuckets }; } + // TODO Duplicated std::map< MpiRank, Exchange > exchange( int commId, std::vector< NeighborCommunicator > & neighbors, @@ -320,6 +377,7 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, MpiRank const curRank{ MpiWrapper::commRank() }; std::vector< NeighborCommunicator > ncs; + ncs.reserve( neighbors.size() ); for( MpiRank const & rank: neighbors ) { ncs.emplace_back( rank.get() ); @@ -329,7 +387,7 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::map< MpiRank, Exchange > exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); - std::map< std::set< MpiRank >, std::set< Edge > > const overlappingEdges = findOverlappingEdges( exchanged, curRank, neighbors ); + Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, @@ -341,7 +399,7 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, neighbors_.insert( MpiRank{ rank } ); } - return doTheNewGhosting(mesh, neighbors_); + return doTheNewGhosting( mesh, neighbors_ ); } } // end of namespace geos::ghosting From 299325ab4b1bce6bf0180df40aeef5ceb02b7bc7 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:44:45 -0700 Subject: [PATCH 020/106] Estimate the max buffer size using an MPI::sum --- .../mesh/generators/NewGhosting.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index f376fbca27a..fe481c097c7 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -370,6 +370,29 @@ std::map< MpiRank, Exchange > exchange( int commId, using ScannedOffsets = std::map< std::set< MpiRank >, integer >; +std::size_t buildMaxBufferSize( Buckets const & buckets ) +{ + std::size_t edgeSize = std::size( buckets.edges ); + for( auto const & [ranks, edges]: buckets.edges ) + { + edgeSize += std::size( ranks ) + 1; + edgeSize += 2 * std::size( edges ) + 1; + } + + std::size_t faceSize = std::size( buckets.faces ); + for( auto const & [ranks, faces]: buckets.faces ) + { + faceSize += std::size( ranks ) + 1; + for( Face const & face: faces ) + { + faceSize = std::size( face ) + 1; + } + } + + std::size_t const total = edgeSize + faceSize; + return MpiWrapper::sum( total ); +} + void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { @@ -388,6 +411,8 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); + + std::size_t const maxBufferSize = buildMaxBufferSize( buckets ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From b26854db9148fa732d572e94103a3247c6e55ede Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:56:59 -0700 Subject: [PATCH 021/106] Wrong impl + first struct to hold the offsets. --- .../mesh/generators/NewGhosting.cpp | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index fe481c097c7..71210cea9a4 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -367,30 +367,29 @@ std::map< MpiRank, Exchange > exchange( int commId, return output; } -using ScannedOffsets = std::map< std::set< MpiRank >, integer >; + +struct ScannedOffsets +{ + std::map< std::set< MpiRank >, EdgeGlbIdx > edgeOffsets; + std::map< std::set< MpiRank >, FaceGlbIdx > faceOffsets; + EdgeGlbIdx edgeRestart; + FaceGlbIdx faceRestart; +}; std::size_t buildMaxBufferSize( Buckets const & buckets ) { - std::size_t edgeSize = std::size( buckets.edges ); - for( auto const & [ranks, edges]: buckets.edges ) + auto const f = []( auto const & bucket ) -> std::size_t { - edgeSize += std::size( ranks ) + 1; - edgeSize += 2 * std::size( edges ) + 1; - } - - std::size_t faceSize = std::size( buckets.faces ); - for( auto const & [ranks, faces]: buckets.faces ) - { - faceSize += std::size( ranks ) + 1; - for( Face const & face: faces ) + std::size_t size = std::size( bucket ); + for( auto const & [ranks, _]: bucket ) { - faceSize = std::size( face ) + 1; + size += std::size( ranks ) + 1 + 1; // One `+1` for the size of the ranks set, the other one for the offset } - } + return size; + }; - std::size_t const total = edgeSize + faceSize; - return MpiWrapper::sum( total ); + return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From ca65b2dc68dcf0e8c66a057afcd8382b7213f1eb Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:13:04 -0700 Subject: [PATCH 022/106] Comment --- src/coreComponents/mesh/generators/NewGhosting.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 71210cea9a4..f56c82baccd 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -389,7 +389,7 @@ std::size_t buildMaxBufferSize( Buckets const & buckets ) return size; }; - return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); + return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); // Add the ScannedOffsets::{edge, face}Restart } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From da4103a298285412e5b0b3d5d1b7211a4e010f09 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:57:19 -0700 Subject: [PATCH 023/106] add json lib for faster dev --- src/coreComponents/mesh/CMakeLists.txt | 9 ++++++++- src/coreComponents/mesh/generators/NewGhosting.cpp | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index 1ad47f2e12f..d0a1fd717ad 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -168,7 +168,14 @@ if( NOT namedtype_POPULATED ) add_subdirectory( ${namedtype_SOURCE_DIR} ${namedtype_BINARY_DIR} ) endif() -list( APPEND dependencyList NamedType ) +include(FetchContent) + +FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz) +FetchContent_MakeAvailable(json) + +#target_link_libraries(foo PRIVATE nlohmann_json::nlohmann_json) + +list( APPEND dependencyList NamedType nlohmann_json::nlohmann_json) if( ENABLE_VTK ) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index f56c82baccd..070c025bfd1 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -28,6 +28,9 @@ #include #include +#include +using json = nlohmann::json; + #include #include From c129c9cf7485f92dbac5232a4fedaa77468aa1a7 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:59:56 -0700 Subject: [PATCH 024/106] use tuple instead of pair for edges --- src/coreComponents/mesh/generators/NewGhosting.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 070c025bfd1..79e290b06c5 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -59,7 +59,7 @@ using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, f // std::vector< NodeGlbIdx > nodes; //}; -using Edge = std::pair< NodeGlbIdx, NodeGlbIdx >; +using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; using Face = std::set< NodeGlbIdx >; struct Exchange @@ -180,8 +180,8 @@ array1d< globalIndex > convertExchange( Exchange const & exchange ) result.emplace_back( std::size( exchange.edges ) ); for( Edge const & edge: exchange.edges ) { - result.emplace_back( edge.first.get() ); - result.emplace_back( edge.second.get() ); + result.emplace_back( std::get< 0 >( edge ).get() ); + result.emplace_back( std::get< 1 >( edge ).get() ); } result.emplace_back( std::size( exchange.faces ) ); for( Face const & face: exchange.faces ) From 38978557ce2f443f9ff5e8f7d214a3f3aabc6a91 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:05:28 -0700 Subject: [PATCH 025/106] save poc --- scripts/poc.cpp | 203 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 scripts/poc.cpp diff --git a/scripts/poc.cpp b/scripts/poc.cpp new file mode 100644 index 00000000000..9a725635b76 --- /dev/null +++ b/scripts/poc.cpp @@ -0,0 +1,203 @@ +/* + * Compile: mpicxx -std=c++20 -I /tmp/json/include scan.cpp -o scan + * Launch: mpirun -n 5 scan + */ + +#include + +using json = nlohmann::json; + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +std::map< std::set< int >, int > get_bucket_sizes( int rank ) +{ + // for rank, intersection in enumerate(intersections): + // print(f" --> {rank}") + // for r, i in sorted(intersection.items()): + // print(f"{{{{{', '.join(map(str, r))}}}, {len(i)}}},") + + switch( rank ) + { + case 0: + return { + { { 0 }, 799 }, + { { 0, 1 }, 70 }, + { { 0, 1, 2 }, 6 }, + { { 0, 1, 2, 3 }, 1 }, + { { 0, 1, 3 }, 6 }, + { { 0, 2 }, 118 }, + { { 0, 2, 3 }, 6 }, + { { 0, 3 }, 47 } }; + case 1: + return { + { { 0, 1 }, 70 }, + { { 0, 1, 2 }, 6 }, + { { 0, 1, 2, 3 }, 1 }, + { { 0, 1, 3 }, 6 }, + { { 1 }, 781 }, + { { 1, 2 }, 45 }, + { { 1, 2, 3 }, 5 }, + { { 1, 3 }, 106 } }; + case 2: + return { + { { 0, 1, 2 }, 6 }, + { { 0, 1, 2, 3 }, 1 }, + { { 0, 2 }, 118 }, + { { 0, 2, 3 }, 6 }, + { { 1, 2 }, 45 }, + { { 1, 2, 3 }, 5 }, + { { 2 }, 769 }, + { { 2, 3 }, 72 } }; + case 3: + return { + { { 0, 1, 2, 3 }, 1 }, + { { 0, 1, 3 }, 6 }, + { { 0, 2, 3 }, 6 }, + { { 0, 3 }, 47 }, + { { 1, 2, 3 }, 5 }, + { { 1, 3 }, 106 }, + { { 2, 3 }, 72 }, + { { 3 }, 799 } }; + default: + std::cout << "WRONG!!!" << std::endl; + return {}; + }; +} + +/** + * `rank` the current rank + */ +std::map< std::set< int >, int > update_bucket_offsets( std::map< std::set< int >, int > const & sizes, + std::map< std::set< int >, int > const & offsets, + int rank ) +{ + std::map< std::set< int >, int > reduced; + + // Only consider the offsets that are still relevant (i.e. with high ranks) + for( auto const & [ranks, offset]: offsets ) + { + int const max_concerned = *std::max_element( ranks.begin(), ranks.cend() ); + if( max_concerned < rank ) + { + continue; + } + reduced.emplace_hint( reduced.end(), ranks, offset ); + } + + // Add the offsets associated to the new buckets + int next_offset = 0; + bool offroad = false; + for( auto const & [ranks, size]: sizes ) + { + auto const it = reduced.find( ranks ); + if( it == reduced.end() ) + { + reduced.emplace_hint( reduced.end(), ranks, next_offset ); + next_offset += size; + } + else + { + next_offset = it->second + size; // Define the new offset from the last + } + } + + // Add an extra entry based for the following rank + reduced.emplace_hint( reduced.end(), std::set< int >{ rank + 1 }, next_offset ); + + return reduced; +} + +std::map< std::set< int >, int > unserialize( void const * data, + int len ) +{ + + std::uint8_t const * data0 = reinterpret_cast(data); + + const std::span< const std::uint8_t > s( data0, len ); + json const j = json::from_cbor( s, false ); + + int rank; + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + + return j.get< std::map< std::set< int >, int>>(); +} + +std::vector< std::uint8_t > serialize( std::map< std::set< int >, int > const & in ) +{ + return json::to_cbor( json( in ) ); +} + +/** + * `inout` contains the input from the current rank and must overwritten with the new result. + * `in` contains the reduced result from the previous ranks + */ +void f( void * in, + void * inout, + int * len, + MPI_Datatype * dptr ) +{ + std::map< std::set< int >, int > offsets = unserialize( in, *len ); // offsets provided by the previous rank(s) + std::map< std::set< int >, int > sizes = unserialize( inout, *len ); // sizes of the buckets provided by the current rank + + int rank; + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + + std::map< std::set< int >, int > updated_offsets = update_bucket_offsets( sizes, offsets, rank ); + + + std::vector< std::uint8_t > const serialized = serialize( updated_offsets ); + std::memcpy( inout, serialized.data(), *len ); +} + +int main( int argc, + char ** argv ) +{ + constexpr int N = 2048; + + int rank, size; + std::uint8_t * local = (std::uint8_t *) ( malloc( N * sizeof( std::uint8_t ) ) ); + std::uint8_t * recv = (std::uint8_t *) ( malloc( N * sizeof( std::uint8_t ) ) ); + + MPI_Init( &argc, &argv ); + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + MPI_Comm_size( MPI_COMM_WORLD, &size ); + + MPI_Op myOp; + MPI_Op_create( f, false, &myOp ); + + // Input + std::map< std::set< int >, int > sizes = get_bucket_sizes( rank ); + if( rank == 0 ) + { + sizes = update_bucket_offsets( sizes, { { { 0, }, 0 } }, rank ); + } + std::vector< std::uint8_t > const mm = serialize( sizes ); + std::memcpy( local, mm.data(), mm.size() ); + + MPI_Scan( local, recv, N, MPI_BYTE, myOp, MPI_COMM_WORLD ); + + std::map< std::set< int >, int > loc = unserialize( local, N ); + std::map< std::set< int >, int > rec = unserialize( recv, N ); + std::cout << "on rank " << rank << " FINAL local, recv -> " << json( loc ) << " | " << json( rec ) << std::endl; + + // Just be a little in order + MPI_Barrier( MPI_COMM_WORLD ); + + MPI_Finalize(); + + free( local ); + free( recv ); + + return 0; +} From 4ba3a646ff02010c57ca9b8d452dd75382d47908 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:38:15 -0700 Subject: [PATCH 026/106] use pointer to instance for local information. --- scripts/poc.cpp | 80 ++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/scripts/poc.cpp b/scripts/poc.cpp index 9a725635b76..cced4ba1315 100644 --- a/scripts/poc.cpp +++ b/scripts/poc.cpp @@ -97,7 +97,6 @@ std::map< std::set< int >, int > update_bucket_offsets( std::map< std::set< int // Add the offsets associated to the new buckets int next_offset = 0; - bool offroad = false; for( auto const & [ranks, size]: sizes ) { auto const it = reduced.find( ranks ); @@ -118,13 +117,9 @@ std::map< std::set< int >, int > update_bucket_offsets( std::map< std::set< int return reduced; } -std::map< std::set< int >, int > unserialize( void const * data, - int len ) +template< class V > +std::map< std::set< int >, int > deserialize( V const & s ) { - - std::uint8_t const * data0 = reinterpret_cast(data); - - const std::span< const std::uint8_t > s( data0, len ); json const j = json::from_cbor( s, false ); int rank; @@ -133,71 +128,96 @@ std::map< std::set< int >, int > unserialize( void const * data, return j.get< std::map< std::set< int >, int>>(); } +std::map< std::set< int >, int > deserialize( std::uint8_t const * data, + int len ) +{ + const std::span< const std::uint8_t > s( data, len ); + return deserialize( s ); +} + std::vector< std::uint8_t > serialize( std::map< std::set< int >, int > const & in ) { return json::to_cbor( json( in ) ); } /** - * `inout` contains the input from the current rank and must overwritten with the new result. * `in` contains the reduced result from the previous ranks + * `inout` contains the input from the current rank and must overwritten with the new result. */ void f( void * in, void * inout, int * len, MPI_Datatype * dptr ) { - std::map< std::set< int >, int > offsets = unserialize( in, *len ); // offsets provided by the previous rank(s) - std::map< std::set< int >, int > sizes = unserialize( inout, *len ); // sizes of the buckets provided by the current rank - int rank; MPI_Comm_rank( MPI_COMM_WORLD, &rank ); - std::map< std::set< int >, int > updated_offsets = update_bucket_offsets( sizes, offsets, rank ); + // offsets provided by the previous rank(s) + std::map< std::set< int >, int > offsets = deserialize( reinterpret_cast(in), *len ); + + // Sizes provided by the current rank, under the form of a pointer to the data. + // No need to serialize, since we're on the same rank. + std::uintptr_t addr; + std::memcpy( &addr, inout, sizeof( std::uintptr_t ) ); + std::map< std::set< int >, int > const * sizes = reinterpret_cast, int > *>(addr); + std::map< std::set< int >, int > updated_offsets = update_bucket_offsets( *sizes, offsets, rank ); + // Serialize the updated offsets, so they get sent to the next rank. std::vector< std::uint8_t > const serialized = serialize( updated_offsets ); - std::memcpy( inout, serialized.data(), *len ); + std::memcpy( inout, serialized.data(), serialized.size() ); } int main( int argc, char ** argv ) { - constexpr int N = 2048; - int rank, size; - std::uint8_t * local = (std::uint8_t *) ( malloc( N * sizeof( std::uint8_t ) ) ); - std::uint8_t * recv = (std::uint8_t *) ( malloc( N * sizeof( std::uint8_t ) ) ); - MPI_Init( &argc, &argv ); MPI_Comm_rank( MPI_COMM_WORLD, &rank ); MPI_Comm_size( MPI_COMM_WORLD, &size ); + constexpr int N = 2048; + + std::vector< std::uint8_t > local( N, 0 ); + std::vector< std::uint8_t > recv( N, 0 ); + MPI_Op myOp; MPI_Op_create( f, false, &myOp ); - // Input - std::map< std::set< int >, int > sizes = get_bucket_sizes( rank ); + // For the rank 0, the `MPI_Scan` will not call the reduction operator. + // So we need to reduce ourselves. Still we need to send this reduction to the following rank, + // by copying to it to the send buffer. + // + // For the other ranks, the reduction operator will be called. + // We'll then provide the data as a pointer to the instance on the current rank. + // The reduction operator will then update the offsets and send them to the following rank. + std::map< std::set< int >, int > const sizes = get_bucket_sizes( rank ); + std::map< std::set< int >, int > result; if( rank == 0 ) { - sizes = update_bucket_offsets( sizes, { { { 0, }, 0 } }, rank ); + result = update_bucket_offsets( sizes, { { { 0, }, 0 } }, rank ); + std::vector< std::uint8_t > const bytes = serialize( result ); + std::memcpy( local.data(), bytes.data(), bytes.size() ); + } + else + { + std::uintptr_t const addr = reinterpret_cast(&sizes); + std::memcpy( local.data(), &addr, sizeof( std::uintptr_t ) ); } - std::vector< std::uint8_t > const mm = serialize( sizes ); - std::memcpy( local, mm.data(), mm.size() ); - MPI_Scan( local, recv, N, MPI_BYTE, myOp, MPI_COMM_WORLD ); + MPI_Scan( local.data(), recv.data(), N, MPI_BYTE, myOp, MPI_COMM_WORLD ); - std::map< std::set< int >, int > loc = unserialize( local, N ); - std::map< std::set< int >, int > rec = unserialize( recv, N ); - std::cout << "on rank " << rank << " FINAL local, recv -> " << json( loc ) << " | " << json( rec ) << std::endl; + if( rank != 0 ) + { + result = deserialize( recv ); + } + + std::cout << "on rank " << rank << " FINAL recv -> " << json( result ) << std::endl; // Just be a little in order MPI_Barrier( MPI_COMM_WORLD ); MPI_Finalize(); - free( local ); - free( recv ); - return 0; } From 321672b7d8e727c3775d3b2ecc5b26fdcdcb8449 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:46:48 -0700 Subject: [PATCH 027/106] change names of buffers --- scripts/poc.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/poc.cpp b/scripts/poc.cpp index cced4ba1315..d9eee7e9be6 100644 --- a/scripts/poc.cpp +++ b/scripts/poc.cpp @@ -178,8 +178,8 @@ int main( int argc, constexpr int N = 2048; - std::vector< std::uint8_t > local( N, 0 ); - std::vector< std::uint8_t > recv( N, 0 ); + std::vector< std::uint8_t > sendBuff( N, 0 ); + std::vector< std::uint8_t > recvBuff( N, 0 ); MPI_Op myOp; MPI_Op_create( f, false, &myOp ); @@ -197,22 +197,22 @@ int main( int argc, { result = update_bucket_offsets( sizes, { { { 0, }, 0 } }, rank ); std::vector< std::uint8_t > const bytes = serialize( result ); - std::memcpy( local.data(), bytes.data(), bytes.size() ); + std::memcpy( sendBuff.data(), bytes.data(), bytes.size() ); } else { std::uintptr_t const addr = reinterpret_cast(&sizes); - std::memcpy( local.data(), &addr, sizeof( std::uintptr_t ) ); + std::memcpy( sendBuff.data(), &addr, sizeof( std::uintptr_t ) ); } - MPI_Scan( local.data(), recv.data(), N, MPI_BYTE, myOp, MPI_COMM_WORLD ); + MPI_Scan( sendBuff.data(), recvBuff.data(), N, MPI_BYTE, myOp, MPI_COMM_WORLD ); if( rank != 0 ) { - result = deserialize( recv ); + result = deserialize( recvBuff ); } - std::cout << "on rank " << rank << " FINAL recv -> " << json( result ) << std::endl; + std::cout << "on rank " << rank << " FINAL offsets -> " << json( result ) << std::endl; // Just be a little in order MPI_Barrier( MPI_COMM_WORLD ); From 650aa23b498c4172bbbe28848ef141e628dbc3d3 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 25 Apr 2024 17:05:12 -0700 Subject: [PATCH 028/106] First OK exchange of the edges offsets --- .../mesh/generators/NewGhosting.cpp | 253 ++++++++++++++++-- 1 file changed, 229 insertions(+), 24 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 79e290b06c5..4a1122200f0 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -40,34 +40,59 @@ namespace geos::ghosting using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; +using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable >; using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; -using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable >; +using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; -//struct Edge -//{ -// std::pair< NodeGlbIdx, NodeGlbIdx > nodes; -//}; -//struct Face -//{ -// std::vector< NodeGlbIdx > nodes; -//}; +void to_json( json & j, + const MpiRank & v ) +{ + j = v.get(); +} + +void from_json( const json & j, + MpiRank & v ) +{ + v = MpiRank{ j.get< MpiRank::UnderlyingType >() }; // TODO use a `traits` instead +} + +void to_json( json & j, + const EdgeLocIdx & v ) +{ + j = v.get(); +} + +void from_json( const json & j, + EdgeLocIdx & v ) +{ + v = EdgeLocIdx{ j.get< EdgeLocIdx::UnderlyingType >() }; +} + +void to_json( json & j, + const FaceLocIdx & v ) +{ + j = v.get(); +} + +void from_json( const json & j, + FaceLocIdx & v ) +{ + v = FaceLocIdx{ j.get< FaceLocIdx::UnderlyingType >() }; +} using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; -using Face = std::set< NodeGlbIdx >; +using Face = std::set< NodeGlbIdx >; // TODO change to std::vector< NodeGlbIdx > with a specific ordering. struct Exchange { std::set< Edge > edges; std::set< Face > faces; -// SortedArray< Edge > edges; -// SortedArray< Face > faces; }; template< bool DO_PACKING > @@ -371,15 +396,6 @@ std::map< MpiRank, Exchange > exchange( int commId, } -struct ScannedOffsets -{ - std::map< std::set< MpiRank >, EdgeGlbIdx > edgeOffsets; - std::map< std::set< MpiRank >, FaceGlbIdx > faceOffsets; - EdgeGlbIdx edgeRestart; - FaceGlbIdx faceRestart; -}; - - std::size_t buildMaxBufferSize( Buckets const & buckets ) { auto const f = []( auto const & bucket ) -> std::size_t @@ -395,6 +411,162 @@ std::size_t buildMaxBufferSize( Buckets const & buckets ) return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); // Add the ScannedOffsets::{edge, face}Restart } +struct BucketSizes +{ + using mapping = std::map< std::set< MpiRank >, localIndex >; + mapping edges; + mapping faces; +}; + +void to_json( json & j, + const BucketSizes & v ) +{ + j = json{ { "edges", v.edges }, { "faces", v.faces } }; +} + +void from_json( const json & j, + BucketSizes & v ) +{ + v.edges = j.at( "edges" ).get< BucketSizes::mapping >(); + v.faces = j.at( "faces" ).get< BucketSizes::mapping >(); +} + +struct BucketOffsets +{ + std::map< std::set< MpiRank >, EdgeLocIdx > edges; + std::map< std::set< MpiRank >, FaceLocIdx > faces; +}; + +void to_json( json & j, + const BucketOffsets & v ) +{ + j = json{ { "edges", v.edges }, { "faces", v.faces } }; +} + +void from_json( const json & j, + BucketOffsets & v ) +{ + v.edges = j.at( "edges" ).get< std::map< std::set< MpiRank >, EdgeLocIdx > >(); + v.faces = j.at( "faces" ).get< std::map< std::set< MpiRank >, FaceLocIdx > >(); +} + +BucketSizes getBucketSize(Buckets const & buckets) +{ + BucketSizes output; + + for( auto const & [ranks, edges]: buckets.edges ) + { + output.edges.emplace_hint( output.edges.end(), ranks, std::size( edges ) ); + } + + for( auto const & [ranks, faces]: buckets.faces ) + { + output.faces.emplace_hint( output.faces.end(), ranks, std::size( faces ) ); + } + + return output; +} + +/** + * @brief + * @tparam LOC_IDX The local index type of the geometrical quantity considered (typically @p EdgeLocIdx or @p FaceLocIdx). + * @param sizes + * @param offsets + * @param curRank + * @return + */ +template< typename LOC_IDX > +std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, + std::map< std::set< MpiRank >, LOC_IDX > const & offsets, + MpiRank curRank ) +{ + std::map< std::set< MpiRank >, LOC_IDX > reducedOffsets; + + // Only consider the offsets that are still relevant (i.e. with high ranks) + for( auto const & [ranks, offset]: offsets ) + { + MpiRank const maxConcerned = *std::max_element( ranks.begin(), ranks.cend() ); + if( maxConcerned < curRank ) + { + continue; + } + reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); + } + + // Add the offsets associated to the new buckets + LOC_IDX nextOffset{ 0 }; + for( auto const & [ranks, size]: sizes ) + { + auto const it = reducedOffsets.find( ranks ); + if( it == reducedOffsets.end() ) + { + reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, nextOffset ); + nextOffset += LOC_IDX{ size }; + } + else + { + nextOffset = it->second + LOC_IDX{ size }; // Define the new offset from the last + } + } + + // Add an extra entry based for the following rank + reducedOffsets.emplace_hint( reducedOffsets.end(), std::set< MpiRank >{ curRank + MpiRank{ 1 } }, nextOffset ); + + return reducedOffsets; +} + +std::vector< std::uint8_t > serialize( BucketSizes const & sizes ) +{ +// std::cout << json( sizes ) << std::endl; + return json::to_cbor( json( sizes ) ); +} + +std::vector< std::uint8_t > serialize( BucketOffsets const & offsets ) +{ +// std::cout << json( offsets ) << std::endl; + return json::to_cbor( json( offsets ) ); +} + +/** + * @brief + * @tparam BUCKET_TYPE + * @tparam V Container of std::uint8_t + * @param data + * @return + */ +template< class V > +BucketOffsets deserialize( V const & data ) +{ + return json::from_cbor( data, false ).template get< BucketOffsets >(); +} + +void f( void * in, + void * inout, + int * len, + MPI_Datatype * dptr ) +{ + GEOS_ASSERT_EQ( *dptr, MPI_BYTE ); + + MpiRank const curRank{ MpiWrapper::commRank() }; + + // offsets provided by the previous rank(s) + BucketOffsets const offsets = deserialize( Span< std::uint8_t >( (std::uint8_t *) in, *len ) ); + + // Sizes provided by the current rank, under the form of a pointer to the data. + // No need to serialize, since we're on the same rank. + std::uintptr_t addr; + std::memcpy( &addr, inout, sizeof( std::uintptr_t ) ); + BucketSizes const * sizes = reinterpret_cast(addr); + + BucketOffsets updatedOffsets; + updatedOffsets.edges = updateBucketOffsets< EdgeLocIdx >( sizes->edges, offsets.edges, curRank ); + updatedOffsets.faces = updateBucketOffsets< FaceLocIdx >( sizes->faces, offsets.faces, curRank ); + + // Serialize the updated offsets, so they get sent to the next rank. + std::vector< std::uint8_t > const serialized = serialize( updatedOffsets ); + std::memcpy( inout, serialized.data(), serialized.size() ); +} + void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { @@ -414,7 +586,40 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); - std::size_t const maxBufferSize = buildMaxBufferSize( buckets ); + std::size_t const maxBufferSize = 100 * buildMaxBufferSize( buckets ); + + std::vector< std::uint8_t > sendBuffer( maxBufferSize ); + std::vector< std::uint8_t > recvBuffer( maxBufferSize ); + + BucketSizes const sizes = getBucketSize( buckets ); + + BucketOffsets offsets; + if( curRank == MpiRank{ 0 } ) + { + offsets.edges = updateBucketOffsets< EdgeLocIdx >( sizes.edges, { { { MpiRank{ 0 }, }, EdgeLocIdx{ 0 } } }, curRank ); + offsets.faces = updateBucketOffsets< FaceLocIdx >( sizes.faces, { { { MpiRank{ 0 }, }, FaceLocIdx{ 0 } } }, curRank ); + std::vector< std::uint8_t > const bytes = serialize( offsets ); + std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); + } + else + { + std::uintptr_t const addr = reinterpret_cast(&sizes); + std::memcpy( sendBuffer.data(), &addr, sizeof( std::uintptr_t ) ); + } + + MPI_Op op; + MPI_Op_create( f, false, &op ); + + MPI_Scan( sendBuffer.data(), recvBuffer.data(), maxBufferSize, MPI_BYTE, op, MPI_COMM_WORLD ); + + if( curRank != MpiRank{ 0 } ) + { + offsets = deserialize( recvBuffer ); + } + + std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; + +// BucketOffsets const receivedOffsets = deserialize< BucketOffsets >( recvBuffer ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From 74e05cc405bef38c54eeb8103165a62069b92b5c Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 25 Apr 2024 18:22:07 -0700 Subject: [PATCH 029/106] Reorder the nodes of the face and comments. --- .../mesh/generators/NewGhosting.cpp | 78 +++++++++++++++---- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 4a1122200f0..5a4d71809ab 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -87,7 +87,7 @@ void from_json( const json & j, } using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; -using Face = std::set< NodeGlbIdx >; // TODO change to std::vector< NodeGlbIdx > with a specific ordering. +using Face = std::vector< NodeGlbIdx >; struct Exchange { @@ -129,6 +129,52 @@ std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) return boundaryCellIdxs; } +/** + * @brief Order the nodes of the faces in a way that can be reproduced across the MPI ranks. + * @param nodes The list of nodes as provided by the mesh. + * @return A face with the nodes in the appropriate order + * @details The nodes will be ordered in the following way. + * First, we look for the lowest node index. It will become the first node. + * Then we must pick the second node. We have to choices: just before or just after the first. + * (we do not want to shuffle the nodes completely, we need to keep track of the order). + * To do this, we select the nodes with the lowest index as well. + * This also defines a direction in which we'll pick the other nodes. + * For example, the face [2, 3, 1, 5, 9, 8] will become [1, 3, 2, 8, 9, 5] + * because we'll start with @c 1 and then select the @c 3 over the @c 5. + * Which defines the direction @c 2, @c 8, @c 9, @c 5. + * @note This is the same pattern that we apply for edges. + * Except that edges having only two nodes, it's not necessary to implement a dedicated function + * and std::minmax is enough. + */ +Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) +{ + std::size_t const n = nodes.size(); + + // Handles negative values of `i`. + auto const modulo = [n]( integer const & i ) -> std::size_t + { + integer mod = i % n; + if( mod < 0 ) + { + mod += n; + } + return mod; + }; + + Face f; + f.reserve( n ); + + auto const it = std::min_element( nodes.cbegin(), nodes.cend() ); + std::size_t const minIdx = std::distance( nodes.cbegin(), it ); + int const increment = nodes[modulo( minIdx - 1 )] < nodes[modulo( minIdx + 1 )] ? -1 : 1; + integer i = minIdx; + for( std::size_t count = 0; count < n; ++count, i = i + increment ) + { + f.emplace_back( nodes.at( modulo( i ) ) ); + } + + return f; +} Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdType > const & cellIds ) @@ -164,21 +210,21 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, vtkIdType const ln1 = edge->GetPointId( 1 ); vtkIdType const gn0 = globalPtIds->GetValue( ln0 ); vtkIdType const gn1 = globalPtIds->GetValue( ln1 ); - tmpEdges.emplace_back( std::minmax( { gn0, gn1 } ) ); + tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ gn0 }, NodeGlbIdx{ gn1 } } ) ); } for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) { vtkCell * face = cell->GetFace( f ); vtkIdList * pids = face->GetPointIds(); - Face ff; - for( auto i = 0; i < pids->GetNumberOfIds(); ++i ) + std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); + for( std::size_t i = 0; i < nodes.size(); ++i ) { vtkIdType const lni = face->GetPointId( i ); vtkIdType const gni = globalPtIds->GetValue( lni ); - ff.insert( NodeGlbIdx{ gni } ); + nodes[i] = NodeGlbIdx{ gni }; } - tmpFaces.emplace_back( ff ); + tmpFaces.emplace_back( reorderFaceNodes( nodes ) ); } } @@ -239,10 +285,10 @@ Exchange convertExchange( array1d< globalIndex > const & input ) for( ++i; i < input.size(); ) { auto const s = input[i++]; - std::set< NodeGlbIdx > face; + Face face( s ); for( int j = 0; j < s; ++j, ++i ) { - face.insert( NodeGlbIdx{ input[i] } ); + face[j] = NodeGlbIdx{ input[i] }; } exchange.faces.insert( face ); } @@ -469,7 +515,7 @@ BucketSizes getBucketSize(Buckets const & buckets) /** * @brief - * @tparam LOC_IDX The local index type of the geometrical quantity considered (typically @p EdgeLocIdx or @p FaceLocIdx). + * @tparam LOC_IDX The local index type of the geometrical quantity considered (typically @c EdgeLocIdx or @c FaceLocIdx). * @param sizes * @param offsets * @param curRank @@ -517,19 +563,16 @@ std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set std::vector< std::uint8_t > serialize( BucketSizes const & sizes ) { -// std::cout << json( sizes ) << std::endl; return json::to_cbor( json( sizes ) ); } std::vector< std::uint8_t > serialize( BucketOffsets const & offsets ) { -// std::cout << json( offsets ) << std::endl; return json::to_cbor( json( offsets ) ); } /** * @brief - * @tparam BUCKET_TYPE * @tparam V Container of std::uint8_t * @param data * @return @@ -592,17 +635,24 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::vector< std::uint8_t > recvBuffer( maxBufferSize ); BucketSizes const sizes = getBucketSize( buckets ); - BucketOffsets offsets; if( curRank == MpiRank{ 0 } ) { + // The `MPI_Scan` process will not call the reduction operator for rank 0. + // So we need to reduce ourselves for ourselves. offsets.edges = updateBucketOffsets< EdgeLocIdx >( sizes.edges, { { { MpiRank{ 0 }, }, EdgeLocIdx{ 0 } } }, curRank ); offsets.faces = updateBucketOffsets< FaceLocIdx >( sizes.faces, { { { MpiRank{ 0 }, }, FaceLocIdx{ 0 } } }, curRank ); + // Still we need to send this reduction to the following rank, by copying to it to the send buffer. std::vector< std::uint8_t > const bytes = serialize( offsets ); std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); } else { + // For the other ranks, the reduction operator will be called during the `Mpi_Scan` process. + // So unlike for rank 0, we do not have to do it ourselves. + // In order to provide the `sizes` to the reduction operator, since `sizes` will only be used on the current ranl, + // we'll provide the information as a pointer to the instance. + // The reduction operator will then compute the new offsets and send them to the following rank. std::uintptr_t const addr = reinterpret_cast(&sizes); std::memcpy( sendBuffer.data(), &addr, sizeof( std::uintptr_t ) ); } @@ -618,8 +668,6 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, } std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; - -// BucketOffsets const receivedOffsets = deserialize< BucketOffsets >( recvBuffer ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From d90608c892960b1cd0171f756172e0832cb7f58d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 26 Apr 2024 20:29:02 -0700 Subject: [PATCH 030/106] Reorder the nodes of the face and comments. --- scripts/mpi_global_offset.cpp | 74 +++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 scripts/mpi_global_offset.cpp diff --git a/scripts/mpi_global_offset.cpp b/scripts/mpi_global_offset.cpp new file mode 100644 index 00000000000..f4f356a470b --- /dev/null +++ b/scripts/mpi_global_offset.cpp @@ -0,0 +1,74 @@ +/* + * Compile: mpicxx -std=c++20 mpi_global_offset.cpp -o /tmp/mpi_global_offset + * Launch: mpirun -n 3 /tmp/mpi_global_offset + */ + +#include + +//#include +//#include +#include + +struct S +{ + std::int32_t n; // Compute offsets and not values... S.edgeOffset, S.faceOffset, S.cellOffset + std::int32_t e; + std::int32_t f; +}; + +void f_impl( S const * in, + S * inout ) +{ + inout->n = std::max( in->n, inout->n ); + inout->e = std::max( in->e, inout->e ); + inout->f = std::max( in->f, inout->f ); +} + +void f( void * in, + void * inout, + int * len, + MPI_Datatype * dptr ) +{ + f_impl( (S const *) in, (S *) inout ); +} + +int main( int argc, + char ** argv ) +{ + int rank, size; + MPI_Init( &argc, &argv ); + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + MPI_Comm_size( MPI_COMM_WORLD, &size ); + + std::cout << "rank " << rank << std::endl; + +// constexpr int count = 3; +// std::array< int, count > const lengths{ 1, 1, 1 }; +// std::array< MPI_Aint, count > const offsets{ offsetof( S, n ), offsetof( S, e ), offsetof( S, f ) }; +// std::array< MPI_Datatype, count > const types{ MPI_INT32_T, MPI_INT32_T, MPI_INT32_T }; +// MPI_Datatype t; +// MPI_Type_create_struct( count, lengths.data(), offsets.data(), types.data(), &t ); +// MPI_Type_commit( &t ); + + MPI_Datatype t; + MPI_Type_contiguous( 3, MPI_INT32_T, &t ); + MPI_Type_commit( &t ); + + MPI_Op myOp; + MPI_Op_create( f, true, &myOp ); + + S s{ rank, 2 * rank, - rank }; + S answer; + + MPI_Allreduce( &s, &answer, 1, t, myOp, MPI_COMM_WORLD ); + +// std::cout << "inp on rank " << rank << " is [" << s.n << ", " << s.e << ", " << s.f << "]" << std::endl; + std::cout << "ans on rank " << rank << " is [" << answer.n << ", " << answer.e << ", " << answer.f << "]" << std::endl; + + // Just be a little in order + MPI_Barrier( MPI_COMM_WORLD ); + + MPI_Finalize(); + + return 0; +} From a9b227d4a49c24d85ae29c4121d121957cef22f8 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 26 Apr 2024 21:13:30 -0700 Subject: [PATCH 031/106] Some doc --- .../mesh/generators/NewGhosting.cpp | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 5a4d71809ab..0d24fc81d70 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -106,7 +106,11 @@ Pack( buffer_unit_type *& buffer, return sizeOfPackedChars; } - +/** + * @brief Extract the cells at the boundary of the mesh. + * @param mesh The vtk mesh. + * @return The vtk cell ids. + */ std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) { auto f = vtkDataSetSurfaceFilter::New(); @@ -324,9 +328,11 @@ struct Buckets /** - * @brief - * @param exchanged + * @brief Compute the intersection between for the ranks based on the information @p exchanged. + * @param exchanged Geometrical information provided by the ranks (including the current rank). + * @param curRank The current MPI rank. * @param neighbors excluding current rank + * @return The intersection buckets. */ Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, MpiRank curRank, @@ -442,7 +448,16 @@ std::map< MpiRank, Exchange > exchange( int commId, } -std::size_t buildMaxBufferSize( Buckets const & buckets ) +/** + * @brief Estimate an upper bound of the serialized size of the size buckets. + * @param buckets + * @return Size in bytes. + * @details MPI_Scan requires a fixed buffer size. + * An a-priori estimation of the maximum size of the buffer leads to crazy amounts of memory + * because of the combinatorial pattern of the rank intersections. + * Therefore, we use an MPI communication to get a better estimation. + */ +std::size_t buildMaxBufferSize( Buckets const & buckets ) // TODO give the size buckets instead? { auto const f = []( auto const & bucket ) -> std::size_t { @@ -496,7 +511,12 @@ void from_json( const json & j, v.faces = j.at( "faces" ).get< std::map< std::set< MpiRank >, FaceLocIdx > >(); } -BucketSizes getBucketSize(Buckets const & buckets) +/** + * @brief Compute the sizes of the intersection buckets. + * @param buckets The intersection buckets. + * @return The buckets of sizes. + */ +BucketSizes getBucketSize( Buckets const & buckets ) { BucketSizes output; @@ -516,10 +536,10 @@ BucketSizes getBucketSize(Buckets const & buckets) /** * @brief * @tparam LOC_IDX The local index type of the geometrical quantity considered (typically @c EdgeLocIdx or @c FaceLocIdx). - * @param sizes - * @param offsets - * @param curRank - * @return + * @param sizes The sizes of the intersection bucket (from the current MPI rank). + * @param offsets The offsets for each intersection bucket (from the MPI_Scan process, i.e. the previous ranks). + * @param curRank Current MPI rank. + * @return The offset buckets, updated with the sizes of the current MPI rank. */ template< typename LOC_IDX > std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, @@ -528,10 +548,18 @@ std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set { std::map< std::set< MpiRank >, LOC_IDX > reducedOffsets; - // Only consider the offsets that are still relevant (i.e. with high ranks) + // We only keep the offsets that are still relevant to the current and higher ranks. + // Note that `reducedOffsets` will be used by the _current_ rank too. + // Therefore, we'll send information that may not be useful to higher ranks. + // This is surely acceptable because the information is tiny. for( auto const & [ranks, offset]: offsets ) { - MpiRank const maxConcerned = *std::max_element( ranks.begin(), ranks.cend() ); + MpiRank const maxConcerned = *std::max_element( ranks.cbegin(), ranks.cend() ); + // TODO invert +// if( maxConcerned >= curRank ) +// { +// reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); +// } if( maxConcerned < curRank ) { continue; @@ -539,7 +567,7 @@ std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); } - // Add the offsets associated to the new buckets + // Add the offsets associated to the new size buckets from the current rank. LOC_IDX nextOffset{ 0 }; for( auto const & [ranks, size]: sizes ) { @@ -583,12 +611,21 @@ BucketOffsets deserialize( V const & data ) return json::from_cbor( data, false ).template get< BucketOffsets >(); } +/** + * @brief Custom MPI reduction function. Merges the sizes of the bucket from current rank into numbering offsets. + * @param[in] in Contains the reduced result from the previous ranks. Needs to be unpacked into a @c BucketOffsets instance. + * @param[inout] inout As an @e input, contains a pointer to the @c BucketSizes instance from the current rank. + * As an @e output, will contained the (packed) instance of @c BucketOffsets after the @c reduction. + * The @e output instance will be used by both current and next ranks. + * @param[in] len Number of data. + * @param[in] dataType Type of data. Mean to be @c MPI_BYTE for the current case. + */ void f( void * in, void * inout, int * len, - MPI_Datatype * dptr ) + MPI_Datatype * dataType ) { - GEOS_ASSERT_EQ( *dptr, MPI_BYTE ); + GEOS_ASSERT_EQ( *dataType, MPI_BYTE ); MpiRank const curRank{ MpiWrapper::commRank() }; From a7f42794b587939fd33ca1cf1e991510821ef35f Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 29 Apr 2024 16:52:40 -0700 Subject: [PATCH 032/106] Gather/reduce the max nodes, edges, faces, cells from all the ranks. --- scripts/mpi_global_offset.cpp | 2 +- .../mesh/generators/NewGhosting.cpp | 124 +++++++++++++++--- 2 files changed, 106 insertions(+), 20 deletions(-) diff --git a/scripts/mpi_global_offset.cpp b/scripts/mpi_global_offset.cpp index f4f356a470b..7fcf9460082 100644 --- a/scripts/mpi_global_offset.cpp +++ b/scripts/mpi_global_offset.cpp @@ -13,7 +13,7 @@ struct S { std::int32_t n; // Compute offsets and not values... S.edgeOffset, S.faceOffset, S.cellOffset std::int32_t e; - std::int32_t f; + std::int32_t f; // add cells for global size. }; void f_impl( S const * in, diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 0d24fc81d70..e7eeef43d5a 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -40,10 +40,10 @@ namespace geos::ghosting using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; -using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; -using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; +using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; +using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; @@ -63,27 +63,39 @@ void from_json( const json & j, } void to_json( json & j, - const EdgeLocIdx & v ) + const NodeGlbIdx & v ) +{ + j = v.get(); +} + +void to_json( json & j, + const EdgeGlbIdx & v ) { j = v.get(); } void from_json( const json & j, - EdgeLocIdx & v ) + EdgeGlbIdx & v ) { - v = EdgeLocIdx{ j.get< EdgeLocIdx::UnderlyingType >() }; + v = EdgeGlbIdx{ j.get< EdgeGlbIdx::UnderlyingType >() }; } void to_json( json & j, - const FaceLocIdx & v ) + const FaceGlbIdx & v ) { j = v.get(); } void from_json( const json & j, - FaceLocIdx & v ) + FaceGlbIdx & v ) { - v = FaceLocIdx{ j.get< FaceLocIdx::UnderlyingType >() }; + v = FaceGlbIdx{ j.get< FaceGlbIdx::UnderlyingType >() }; +} + +void to_json( json & j, + const CellGlbIdx & v ) +{ + j = v.get(); } using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; @@ -494,8 +506,8 @@ void from_json( const json & j, struct BucketOffsets { - std::map< std::set< MpiRank >, EdgeLocIdx > edges; - std::map< std::set< MpiRank >, FaceLocIdx > faces; + std::map< std::set< MpiRank >, EdgeGlbIdx > edges; + std::map< std::set< MpiRank >, FaceGlbIdx > faces; }; void to_json( json & j, @@ -507,8 +519,8 @@ void to_json( json & j, void from_json( const json & j, BucketOffsets & v ) { - v.edges = j.at( "edges" ).get< std::map< std::set< MpiRank >, EdgeLocIdx > >(); - v.faces = j.at( "faces" ).get< std::map< std::set< MpiRank >, FaceLocIdx > >(); + v.edges = j.at( "edges" ).get< std::map< std::set< MpiRank >, EdgeGlbIdx > >(); + v.faces = j.at( "faces" ).get< std::map< std::set< MpiRank >, FaceGlbIdx > >(); } /** @@ -639,14 +651,84 @@ void f( void * in, BucketSizes const * sizes = reinterpret_cast(addr); BucketOffsets updatedOffsets; - updatedOffsets.edges = updateBucketOffsets< EdgeLocIdx >( sizes->edges, offsets.edges, curRank ); - updatedOffsets.faces = updateBucketOffsets< FaceLocIdx >( sizes->faces, offsets.faces, curRank ); + updatedOffsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes->edges, offsets.edges, curRank ); + updatedOffsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes->faces, offsets.faces, curRank ); // Serialize the updated offsets, so they get sent to the next rank. std::vector< std::uint8_t > const serialized = serialize( updatedOffsets ); std::memcpy( inout, serialized.data(), serialized.size() ); } +struct MaxGlbIdcs +{ + NodeGlbIdx nodes; + EdgeGlbIdx edges; + FaceGlbIdx faces; + CellGlbIdx cells; +}; + +void to_json( json & j, + const MaxGlbIdcs & mo ) +{ + j = json{ { "nodes", mo.nodes }, + { "edges", mo.edges }, + { "faces", mo.faces }, + { "cells", mo.cells } }; +} + +void g( void * in, + void * inout, + int * len, + MPI_Datatype * dataType ) +{ + GEOS_ASSERT_EQ( *len, 1 ); + + MaxGlbIdcs const * i = reinterpret_cast(in); + MaxGlbIdcs * io = reinterpret_cast(inout); + + io->nodes = std::max( i->nodes, io->nodes ); + io->edges = std::max( i->edges, io->edges ); + io->faces = std::max( i->faces, io->faces ); + io->cells = std::max( i->cells, io->cells ); +} + +MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, + EdgeGlbIdx const & maxEdgeId, + FaceGlbIdx const & maxFaceId ) +{ + MaxGlbIdcs offsets{ NodeGlbIdx{ 0 }, maxEdgeId, maxFaceId, CellGlbIdx{ 0 } }; + + auto const extract = []( vtkDataArray * globalIds ) -> vtkIdType + { + vtkIdTypeArray * gids = vtkIdTypeArray::FastDownCast( globalIds ); + Span< vtkIdType > const s( (vtkIdType *) gids->GetPointer( 0 ), gids->GetNumberOfTuples() ); + return *std::max_element( s.begin(), s.end() ); + }; + + offsets.nodes = NodeGlbIdx{ extract( mesh->GetPointData()->GetGlobalIds() ) }; + offsets.cells = CellGlbIdx{ extract( mesh->GetCellData()->GetGlobalIds() ) }; + + // Otherwise, use `MPI_Type_create_struct`. + static_assert( std::is_same_v< NodeGlbIdx::UnderlyingType, EdgeGlbIdx::UnderlyingType > ); + static_assert( std::is_same_v< NodeGlbIdx::UnderlyingType, FaceGlbIdx::UnderlyingType > ); + static_assert( std::is_same_v< NodeGlbIdx::UnderlyingType, CellGlbIdx::UnderlyingType > ); + +// MPI_Datatype const underlying = internal::getMpiType< NodeGlbIdx::UnderlyingType >(); + MPI_Datatype t; +// MPI_Type_contiguous( sizeof( MatrixOffsets ) / sizeof( underlying ), underlying, &t ); + MPI_Type_contiguous( 4, MPI_LONG_LONG_INT, &t ); + MPI_Type_commit( &t ); + + MPI_Op op; + MPI_Op_create( g, true, &op ); + + MaxGlbIdcs result( offsets ); + + MPI_Allreduce( &offsets, &result, 1, t, op, MPI_COMM_WORLD ); + + return result; +} + void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { @@ -677,8 +759,8 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, { // The `MPI_Scan` process will not call the reduction operator for rank 0. // So we need to reduce ourselves for ourselves. - offsets.edges = updateBucketOffsets< EdgeLocIdx >( sizes.edges, { { { MpiRank{ 0 }, }, EdgeLocIdx{ 0 } } }, curRank ); - offsets.faces = updateBucketOffsets< FaceLocIdx >( sizes.faces, { { { MpiRank{ 0 }, }, FaceLocIdx{ 0 } } }, curRank ); + offsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes.edges, { { { MpiRank{ 0 }, }, EdgeGlbIdx { 0 } } }, curRank ); + offsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes.faces, { { { MpiRank{ 0 }, }, FaceGlbIdx { 0 } } }, curRank ); // Still we need to send this reduction to the following rank, by copying to it to the send buffer. std::vector< std::uint8_t > const bytes = serialize( offsets ); std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); @@ -704,7 +786,11 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, offsets = deserialize( recvBuffer ); } - std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; +// std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; + + MpiRank const nextRank = curRank + MpiRank{ 1 }; + MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ), offsets.faces.at( { nextRank } ) ); + std::cout << "matrixOffsets on rank " << curRank << " -> " << json( matrixOffsets ) << std::endl; } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From f0e6d87a089d18bb6d1e91adf97c3899477daf6c Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 29 Apr 2024 18:24:02 -0700 Subject: [PATCH 033/106] Change template type name. --- .../mesh/generators/NewGhosting.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index e7eeef43d5a..25fa18cfedd 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -547,18 +547,18 @@ BucketSizes getBucketSize( Buckets const & buckets ) /** * @brief - * @tparam LOC_IDX The local index type of the geometrical quantity considered (typically @c EdgeLocIdx or @c FaceLocIdx). + * @tparam GLB_IDX The global index type of the geometrical quantity considered (typically @c EdgeGlbIdx or @c FaceGlbIdx). * @param sizes The sizes of the intersection bucket (from the current MPI rank). * @param offsets The offsets for each intersection bucket (from the MPI_Scan process, i.e. the previous ranks). * @param curRank Current MPI rank. * @return The offset buckets, updated with the sizes of the current MPI rank. */ -template< typename LOC_IDX > -std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, - std::map< std::set< MpiRank >, LOC_IDX > const & offsets, +template< typename GLB_IDX > +std::map< std::set< MpiRank >, GLB_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, + std::map< std::set< MpiRank >, GLB_IDX > const & offsets, MpiRank curRank ) { - std::map< std::set< MpiRank >, LOC_IDX > reducedOffsets; + std::map< std::set< MpiRank >, GLB_IDX > reducedOffsets; // We only keep the offsets that are still relevant to the current and higher ranks. // Note that `reducedOffsets` will be used by the _current_ rank too. @@ -580,18 +580,18 @@ std::map< std::set< MpiRank >, LOC_IDX > updateBucketOffsets( std::map< std::set } // Add the offsets associated to the new size buckets from the current rank. - LOC_IDX nextOffset{ 0 }; + GLB_IDX nextOffset{ 0 }; for( auto const & [ranks, size]: sizes ) { auto const it = reducedOffsets.find( ranks ); if( it == reducedOffsets.end() ) { reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, nextOffset ); - nextOffset += LOC_IDX{ size }; + nextOffset += GLB_IDX{ size }; } else { - nextOffset = it->second + LOC_IDX{ size }; // Define the new offset from the last + nextOffset = it->second + GLB_IDX{ size }; // Define the new offset from the last } } From 3e75ddd80f59bd2bb20689ed89b6f4151ab7558a Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:15:56 -0700 Subject: [PATCH 034/106] Build graph connection as a structure --- .../mesh/generators/NewGhosting.cpp | 88 ++++++++++++++++++- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 25fa18cfedd..003273e2620 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -41,9 +41,9 @@ namespace geos::ghosting using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; +using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::PreIncrementable >; using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; -using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable >; +using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::PreIncrementable >; using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; @@ -182,7 +182,7 @@ Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) auto const it = std::min_element( nodes.cbegin(), nodes.cend() ); std::size_t const minIdx = std::distance( nodes.cbegin(), it ); - int const increment = nodes[modulo( minIdx - 1 )] < nodes[modulo( minIdx + 1 )] ? -1 : 1; + int const increment = nodes[modulo( minIdx - 1 )] < nodes[modulo( minIdx + 1 )] ? -1 : 1; // TODO based on increment, I can say if the face is flipped or not. integer i = minIdx; for( std::size_t count = 0; count < n; ++count, i = i + increment ) { @@ -292,7 +292,7 @@ Exchange convertExchange( array1d< globalIndex > const & input ) int i; for( i = 1; i < 2 * numEdges + 1; i += 2 ) { - NodeGlbIdx gn0{ input[i] }, gn1{ input[i + 1] }; + NodeGlbIdx const gn0{ input[i] }, gn1{ input[i + 1] }; exchange.edges.insert( std::minmax( gn0, gn1 ) ); } GEOS_ASSERT_EQ( std::size_t(numEdges), exchange.edges.size() ); @@ -729,6 +729,84 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, return result; } +struct MeshGraph // TODO add the local <-> global mappings here? +{ + std::map< CellGlbIdx, std::set< FaceGlbIdx > > c2f; // TODO What about the metadata (e.g. flip the face) + std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; // TODO use Face here? + std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? +}; + +MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a sub-mesh? + Buckets const & buckets, + BucketSizes const & sizes, + BucketOffsets const & offsets ) +{ + MeshGraph result; + + for( auto const & [ranks, size]: sizes.edges ) + { + EdgeGlbIdx i{ offsets.edges.at( ranks ) }; + for( Edge const & edge: buckets.edges.at( ranks ) ) + { + result.e2n[i] = edge; + ++i; + } + } + + // Simple inversion + std::map< std::tuple< NodeGlbIdx, NodeGlbIdx >, EdgeGlbIdx > n2e; + for( auto const & [e, n]: result.e2n ) + { + n2e[n] = e; + } + + std::map< std::vector< NodeGlbIdx >, FaceGlbIdx > n2f; + for( auto const & [ranks, size]: sizes.faces ) + { + FaceGlbIdx i{ offsets.faces.at( ranks ) }; + for( Face face: buckets.faces.at( ranks ) ) // Intentional copy for the future `emplace_back`. + { + n2f[face] = i; + face.emplace_back( face.front() ); + for( std::size_t ii = 0; ii < face.size() - 1; ++ii ) + { + NodeGlbIdx const & n0 = face[ii], & n1 = face[ii + 1]; + std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); + std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); + result.f2e[i].insert( n2e.at( p1 ) ); + bool const flipped = p0 != p1; // TODO store somewhere. + } + ++i; + } + } + + vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); + vtkIdTypeArray const * globalCellIds = vtkIdTypeArray::FastDownCast( mesh->GetCellData()->GetGlobalIds() ); // TODO do the mapping beforehand + for( vtkIdType c = 0; c < mesh->GetNumberOfCells(); ++c ) + { + vtkCell * cell = mesh->GetCell( c ); + CellGlbIdx const gci{ globalCellIds->GetValue( c ) }; + // TODO copy paste? + for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) + { + vtkCell * face = cell->GetFace( f ); + vtkIdList * pids = face->GetPointIds(); + std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); + for( std::size_t i = 0; i < nodes.size(); ++i ) + { + vtkIdType const lni = face->GetPointId( i ); + vtkIdType const gni = globalPtIds->GetValue( lni ); + nodes[i] = NodeGlbIdx{ gni }; + } + std::vector< NodeGlbIdx > const reorderedNodes = reorderFaceNodes( nodes ); + result.c2f[gci].insert( n2f.at( reorderedNodes ) ); + // TODO... bool const flipped = ... compare nodes and reorderedNodes. Or ask `reorderFaceNodes` to tell + } + } + + return result; +} + void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { @@ -791,6 +869,8 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, MpiRank const nextRank = curRank + MpiRank{ 1 }; MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ), offsets.faces.at( { nextRank } ) ); std::cout << "matrixOffsets on rank " << curRank << " -> " << json( matrixOffsets ) << std::endl; + + MeshGraph const graph = buildMeshGraph( mesh, buckets, sizes, offsets ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From cb77e630c3efaa3c6b94d4201b7f4cab75784280 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 3 May 2024 10:32:14 -0700 Subject: [PATCH 035/106] Change the serialization of Exchange (use json) --- .../mesh/generators/NewGhosting.cpp | 92 ++++++------------- 1 file changed, 27 insertions(+), 65 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 003273e2620..2927674810f 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -62,6 +62,12 @@ void from_json( const json & j, v = MpiRank{ j.get< MpiRank::UnderlyingType >() }; // TODO use a `traits` instead } +void from_json( const json & j, + NodeGlbIdx & v ) +{ + v = NodeGlbIdx{ j.get< NodeGlbIdx::UnderlyingType >() }; +} + void to_json( json & j, const NodeGlbIdx & v ) { @@ -107,15 +113,17 @@ struct Exchange std::set< Face > faces; }; -template< bool DO_PACKING > -localIndex -Pack( buffer_unit_type *& buffer, - Exchange const & exchange ) +void to_json( json & j, + const Exchange & v ) +{ + j = json{ { "edges", v.edges }, { "faces", v.faces } }; +} + +void from_json( const json & j, + Exchange & v ) { - localIndex sizeOfPackedChars = 0; - sizeOfPackedChars += Pack< DO_PACKING >( buffer, exchange.edges ); - sizeOfPackedChars += Pack< DO_PACKING >( buffer, exchange.faces ); - return sizeOfPackedChars; + v.edges = j.at( "edges" ).get< std::set< Edge > >(); + v.faces = j.at( "faces" ).get< std::set< Face > >(); } /** @@ -251,70 +259,24 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, return { std::move( edges ), std::move( faces ) }; } - -array1d< globalIndex > convertExchange( Exchange const & exchange ) +array1d< std::uint8_t > convertExchange( Exchange const & exchange ) { - std::size_t const edgeSize = 1 + 2 * std::size( exchange.edges ); - std::size_t faceSize = 1; - for( Face const & face: exchange.faces ) - { - faceSize += 1 + std::size( face ); // `+1` because we need to store the size so we know where to stop. - } - - array1d< globalIndex > result; - result.reserve( edgeSize + faceSize ); - - result.emplace_back( std::size( exchange.edges ) ); - for( Edge const & edge: exchange.edges ) - { - result.emplace_back( std::get< 0 >( edge ).get() ); - result.emplace_back( std::get< 1 >( edge ).get() ); - } - result.emplace_back( std::size( exchange.faces ) ); - for( Face const & face: exchange.faces ) + std::vector< std::uint8_t > const tmp = json::to_cbor( json( exchange ) ); + array1d< std::uint8_t > result; + result.reserve( tmp.size() ); + for( std::uint8_t const & t: tmp ) { - result.emplace_back( std::size( face ) ); - for( NodeGlbIdx const & n: face ) - { - result.emplace_back( n.get() ); - } + result.emplace_back( t ); } - return result; } - -Exchange convertExchange( array1d< globalIndex > const & input ) +Exchange convertExchange( array1d< std::uint8_t > const & input ) { - Exchange exchange; - - globalIndex const numEdges = input[0]; - int i; - for( i = 1; i < 2 * numEdges + 1; i += 2 ) - { - NodeGlbIdx const gn0{ input[i] }, gn1{ input[i + 1] }; - exchange.edges.insert( std::minmax( gn0, gn1 ) ); - } - GEOS_ASSERT_EQ( std::size_t(numEdges), exchange.edges.size() ); - - globalIndex const numFaces = input[i]; - for( ++i; i < input.size(); ) - { - auto const s = input[i++]; - Face face( s ); - for( int j = 0; j < s; ++j, ++i ) - { - face[j] = NodeGlbIdx{ input[i] }; - } - exchange.faces.insert( face ); - } - - GEOS_ASSERT_EQ( std::size_t(numFaces), exchange.faces.size() ); - - return exchange; + std::vector< std::uint8_t > const tmp( std::cbegin( input ), std::cend( input ) ); + return json::from_cbor( tmp, false ).template get< Exchange >(); } - Exchange buildFullData( vtkSmartPointer< vtkDataSet > mesh ) { std::set< vtkIdType > cellIds; @@ -423,7 +385,7 @@ std::map< MpiRank, Exchange > exchange( int commId, commData.resize( numNeighbors ); - array1d< globalIndex > const cv = convertExchange( data ); + array1d< std::uint8_t > const cv = convertExchange( data ); for( integer i = 0; i < numNeighbors; ++i ) { @@ -437,7 +399,7 @@ std::map< MpiRank, Exchange > exchange( int commId, MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus() ); MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus() ); - array1d< array1d< globalIndex > > rawExchanged( neighbors.size() ); + array1d< array1d< std::uint8_t > > rawExchanged( neighbors.size() ); for( integer i = 0; i < numNeighbors; ++i ) { From 6b1eda93cb134a339581c7fc89dd70b3cf6ef9f0 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 3 May 2024 10:59:39 -0700 Subject: [PATCH 036/106] Add nodes to the Exchange --- .../mesh/generators/NewGhosting.cpp | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 2927674810f..722f0c33e43 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -109,6 +109,7 @@ using Face = std::vector< NodeGlbIdx >; struct Exchange { + std::set< NodeGlbIdx > nodes; std::set< Edge > edges; std::set< Face > faces; }; @@ -116,12 +117,13 @@ struct Exchange void to_json( json & j, const Exchange & v ) { - j = json{ { "edges", v.edges }, { "faces", v.faces } }; + j = json{ { "nodes", v.nodes }, { "edges", v.edges }, { "faces", v.faces } }; } void from_json( const json & j, Exchange & v ) { + v.nodes = j.at( "nodes" ).get< std::set< NodeGlbIdx > >(); v.edges = j.at( "edges" ).get< std::set< Edge > >(); v.faces = j.at( "faces" ).get< std::set< Face > >(); } @@ -203,7 +205,8 @@ Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdType > const & cellIds ) { - vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); + vtkIdTypeArray * gids = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); + Span< vtkIdType > const globalPtIds( (vtkIdType *) gids->GetPointer( 0 ), gids->GetNumberOfTuples() ); std::vector< Edge > tmpEdges; std::vector< Face > tmpFaces; @@ -232,8 +235,8 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, vtkCell * edge = cell->GetEdge( e ); vtkIdType const ln0 = edge->GetPointId( 0 ); vtkIdType const ln1 = edge->GetPointId( 1 ); - vtkIdType const gn0 = globalPtIds->GetValue( ln0 ); - vtkIdType const gn1 = globalPtIds->GetValue( ln1 ); + vtkIdType const gn0 = globalPtIds[ln0]; + vtkIdType const gn1 = globalPtIds[ln1]; tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ gn0 }, NodeGlbIdx{ gn1 } } ) ); } @@ -245,18 +248,23 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, for( std::size_t i = 0; i < nodes.size(); ++i ) { vtkIdType const lni = face->GetPointId( i ); - vtkIdType const gni = globalPtIds->GetValue( lni ); + vtkIdType const gni = globalPtIds[lni]; nodes[i] = NodeGlbIdx{ gni }; } tmpFaces.emplace_back( reorderFaceNodes( nodes ) ); } } + std::set< NodeGlbIdx > nodes; + std::transform( std::cbegin( globalPtIds ), std::cend( globalPtIds ), + std::inserter( nodes, std::end( nodes ) ), []( vtkIdType const & id ) + { return NodeGlbIdx{ id }; } ); + // Removing the duplicates by copying into a `std::set`. std::set< Edge > edges{ tmpEdges.cbegin(), tmpEdges.cend() }; // SortedArray requires the input to be sorted already. std::set< Face > faces{ tmpFaces.cbegin(), tmpFaces.cend() }; - return { std::move( edges ), std::move( faces ) }; + return { std::move( nodes ), std::move( edges ), std::move( faces ) }; } array1d< std::uint8_t > convertExchange( Exchange const & exchange ) @@ -786,6 +794,8 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::map< MpiRank, Exchange > exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); +// std::cout << "exchanged on rank " << curRank << " -> " << json( exchanged[MpiRank{ MpiWrapper::commRank() }] ) << std::endl; + Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); std::size_t const maxBufferSize = 100 * buildMaxBufferSize( buckets ); From 53497d5861aba5b5b767cc465cede144124fc9d5 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 3 May 2024 12:06:56 -0700 Subject: [PATCH 037/106] Compute the rank intersection for the nodes as well. --- .../mesh/generators/NewGhosting.cpp | 87 ++++++++++++++----- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 722f0c33e43..54a56af2c98 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -304,6 +304,7 @@ Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) struct Buckets { + std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > nodes; std::map< std::set< MpiRank >, std::set< Edge > > edges; std::map< std::set< MpiRank >, std::set< Face > > faces; }; @@ -320,33 +321,67 @@ Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchange MpiRank curRank, std::set< MpiRank > const & neighbors ) { - std::map< Edge, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? - // We "register" all the edges of the current rank: they are the only one we're interested in. - for( Edge const & edge: exchanged.at( curRank ).edges ) - { - counts.emplace_hint( counts.end(), edge, std::set< MpiRank >{ curRank } ); - } + std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > nodeBuckets; + { // Scope reduction // TODO DUPLICATED! + std::map< NodeGlbIdx, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? + Duplicated + // We "register" all the edges of the current rank: they are the only one we're interested in. + for( NodeGlbIdx const & node: exchanged.at( curRank ).nodes ) + { + counts.emplace_hint( counts.end(), node, std::set< MpiRank >{ curRank } ); + } - // We now loop on the neighbor edges. - // If a neighbor has an edge in common with the current rank, they we store it. - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. - { - for( Edge const & edge: exchanged.at( neighborRank ).edges ) + // We now loop on the neighbor edges. + // If a neighbor has an edge in common with the current rank, they we store it. + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + { + for( NodeGlbIdx const & node: exchanged.at( neighborRank ).nodes ) + { + auto it = counts.find( node ); + if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. + { + it->second.insert( neighborRank ); + } + } + } + + for( auto const & [node, ranks]: counts ) { - auto it = counts.find( edge ); - if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. + if( ranks.find( curRank ) != ranks.cend() ) { - it->second.insert( neighborRank ); + nodeBuckets[ranks].insert( node ); } } } std::map< std::set< MpiRank >, std::set< Edge > > edgeBuckets; - for( auto const & [edge, ranks]: counts ) - { - if( ranks.find( curRank ) != ranks.cend() ) + { // Scope reduction + std::map< Edge, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? + // We "register" all the edges of the current rank: they are the only one we're interested in. + for( Edge const & edge: exchanged.at( curRank ).edges ) { - edgeBuckets[ranks].insert( edge ); + counts.emplace_hint( counts.end(), edge, std::set< MpiRank >{ curRank } ); + } + + // We now loop on the neighbor edges. + // If a neighbor has an edge in common with the current rank, they we store it. + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + { + for( Edge const & edge: exchanged.at( neighborRank ).edges ) + { + auto it = counts.find( edge ); + if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. + { + it->second.insert( neighborRank ); + } + } + } + + for( auto const & [edge, ranks]: counts ) + { + if( ranks.find( curRank ) != ranks.cend() ) + { + edgeBuckets[ranks].insert( edge ); + } } } @@ -368,6 +403,13 @@ Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchange // Checking if neighbors is too wide... // TODO do we care? std::set< MpiRank > usefulNeighbors; + for( auto const & [ranks, nodes]: nodeBuckets ) + { + if( not nodes.empty() ) + { + usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); + } + } for( auto const & [ranks, edges]: edgeBuckets ) { if( not edges.empty() ) @@ -379,7 +421,7 @@ Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchange std::set_difference( neighbors.cbegin(), neighbors.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); // TODO... Remove the neighbors? - return { edgeBuckets, faceBuckets }; + return { nodeBuckets, edgeBuckets, faceBuckets }; } @@ -431,13 +473,14 @@ std::map< MpiRank, Exchange > exchange( int commId, /** - * @brief Estimate an upper bound of the serialized size of the size buckets. + * @brief Estimate an upper bound of the serialized size of the size @p buckets. * @param buckets * @return Size in bytes. - * @details MPI_Scan requires a fixed buffer size. + * @details @c MPI_Scan requires a fixed buffer size. * An a-priori estimation of the maximum size of the buffer leads to crazy amounts of memory * because of the combinatorial pattern of the rank intersections. - * Therefore, we use an MPI communication to get a better estimation. + * Therefore, we use an initial MPI communication to get a better estimation. + * @node We don't care about the @c nodes because their global numbering is already done. */ std::size_t buildMaxBufferSize( Buckets const & buckets ) // TODO give the size buckets instead? { From 1c28df078288935fb00c9aa74cb21f53b4855ced Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 3 May 2024 13:15:12 -0700 Subject: [PATCH 038/106] Refactor duplicated code --- .../mesh/generators/NewGhosting.cpp | 116 +++++++++--------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 54a56af2c98..0a2aa67510d 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -311,80 +311,78 @@ struct Buckets /** - * @brief Compute the intersection between for the ranks based on the information @p exchanged. - * @param exchanged Geometrical information provided by the ranks (including the current rank). - * @param curRank The current MPI rank. - * @param neighbors excluding current rank - * @return The intersection buckets. + * @brief + * @tparam T Typically NodeGloIdx or a container of NodeGlbIdx (for edges and faces) + * @param exchanged + * @param curRank + * @param neighbors + * @return */ -Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, - MpiRank curRank, - std::set< MpiRank > const & neighbors ) -{ - std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > nodeBuckets; - { // Scope reduction // TODO DUPLICATED! - std::map< NodeGlbIdx, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? + Duplicated - // We "register" all the edges of the current rank: they are the only one we're interested in. - for( NodeGlbIdx const & node: exchanged.at( curRank ).nodes ) - { - counts.emplace_hint( counts.end(), node, std::set< MpiRank >{ curRank } ); - } +template< typename T > +std::map< std::set< MpiRank >, std::set< T > > +buildIntersectionBuckets( std::map< MpiRank, std::set< T > const & > const & exchanged, + MpiRank curRank, + std::set< MpiRank > const & neighbors ) +{ + std::map< T, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? + // We "register" all the edges of the current rank: they are the only one we're interested in. + for( T const & node: exchanged.at( curRank ) ) + { + counts.emplace_hint( counts.end(), node, std::set< MpiRank >{ curRank } ); + } - // We now loop on the neighbor edges. - // If a neighbor has an edge in common with the current rank, they we store it. - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + // We now loop on the neighbor edges. + // If a neighbor has an edge in common with the current rank, they we store it. + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + { + for( T const & node: exchanged.at( neighborRank ) ) { - for( NodeGlbIdx const & node: exchanged.at( neighborRank ).nodes ) + auto it = counts.find( node ); + if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. { - auto it = counts.find( node ); - if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. - { - it->second.insert( neighborRank ); - } + it->second.insert( neighborRank ); } } + } - for( auto const & [node, ranks]: counts ) + std::map< std::set< MpiRank >, std::set< T > > nodeBuckets; + for( auto const & [node, ranks]: counts ) + { + if( ranks.find( curRank ) != ranks.cend() ) { - if( ranks.find( curRank ) != ranks.cend() ) - { - nodeBuckets[ranks].insert( node ); - } + nodeBuckets[ranks].insert( node ); } } + return nodeBuckets; +} - std::map< std::set< MpiRank >, std::set< Edge > > edgeBuckets; - { // Scope reduction - std::map< Edge, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? - // We "register" all the edges of the current rank: they are the only one we're interested in. - for( Edge const & edge: exchanged.at( curRank ).edges ) - { - counts.emplace_hint( counts.end(), edge, std::set< MpiRank >{ curRank } ); - } - // We now loop on the neighbor edges. - // If a neighbor has an edge in common with the current rank, they we store it. - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. - { - for( Edge const & edge: exchanged.at( neighborRank ).edges ) - { - auto it = counts.find( edge ); - if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. - { - it->second.insert( neighborRank ); - } - } - } +/** + * @brief Compute the intersection between for the ranks based on the information @p exchanged. + * @param exchanged Geometrical information provided by the ranks (including the current rank). + * @param curRank The current MPI rank. + * @param neighbors excluding current rank + * @return The intersection buckets. + */ +Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, + MpiRank curRank, + std::set< MpiRank > const & neighbors ) +{ + std::map< MpiRank, std::set< NodeGlbIdx > const & > nodeInfo; + for( auto const & [rank, exchange]: exchanged ) + { + nodeInfo.emplace( rank, exchange.nodes ); + } + std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > const nodeBuckets = buildIntersectionBuckets( nodeInfo, curRank, neighbors ); - for( auto const & [edge, ranks]: counts ) - { - if( ranks.find( curRank ) != ranks.cend() ) - { - edgeBuckets[ranks].insert( edge ); - } - } + std::map< MpiRank, std::set< Edge > const & > edgeInfo; + for( auto const & [rank, exchange]: exchanged ) + { + edgeInfo.emplace( rank, exchange.edges ); } + std::map< std::set< MpiRank >, std::set< Edge > > const edgeBuckets = buildIntersectionBuckets( edgeInfo, curRank, neighbors ); + // For faces, the algorithm can be a tad simpler because faces can be shared by at most 2 ranks. std::map< std::set< MpiRank >, std::set< Face > > faceBuckets; std::set< Face > curFaces = exchanged.at( curRank ).faces; for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. From 406d0b3bde47375c52c201c57dbeb6a41a437b3a Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 3 May 2024 16:41:26 -0700 Subject: [PATCH 039/106] Build the mesh graph for owned entities. --- .../mesh/generators/NewGhosting.cpp | 129 ++++++++++++++++-- 1 file changed, 115 insertions(+), 14 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 0a2aa67510d..79d9158cf38 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -21,6 +21,8 @@ #include "common/MpiWrapper.hpp" #include "common/DataTypes.hpp" +//#include <_hypre_parcsr_mv.h> + #include #include @@ -309,6 +311,11 @@ struct Buckets std::map< std::set< MpiRank >, std::set< Face > > faces; }; +//void to_json( json & j, +// const Buckets & v ) +//{ +// j = json{ { "nodes", v.nodes }, { "edges", v.edges }, { "faces", v.faces } }; +//} /** * @brief @@ -743,21 +750,60 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, struct MeshGraph // TODO add the local <-> global mappings here? { std::map< CellGlbIdx, std::set< FaceGlbIdx > > c2f; // TODO What about the metadata (e.g. flip the face) - std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; // TODO use Face here? + std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? + // TODO add the nodes here? + // TODO add all types of connections here? How? }; +void to_json( json & j, + const MeshGraph & v ) // For display +{ + j = json{ { "c2f", v.f2e }, { "f2e", v.f2e }, { "e2n", v.e2n } }; +} + + +/** + * @brief Builds the graph information for the owned elements only. + * @param mesh + * @param buckets + * @param offsets + * @param curRank + * @return + */ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a sub-mesh? Buckets const & buckets, - BucketSizes const & sizes, - BucketOffsets const & offsets ) + BucketOffsets const & offsets, + MpiRank curRank ) { MeshGraph result; - for( auto const & [ranks, size]: sizes.edges ) + auto const owning = [&curRank]( std::set< MpiRank > const & ranks ) -> bool { - EdgeGlbIdx i{ offsets.edges.at( ranks ) }; - for( Edge const & edge: buckets.edges.at( ranks ) ) + return curRank == *std::min_element( std::cbegin( ranks ), std::cend( ranks ) ); + }; + + // The `e2n` is a mapping for all the geometrical entities, not only the one owned like `result.e2n`. + // TODO check that it is really useful. + std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; + + for( auto const & [ranks, edges]: buckets.edges ) + { + EdgeGlbIdx i = offsets.edges.at( ranks ); // TODO hack + for( Edge const & edge: edges ) + { + e2n[i] = edge; + ++i; + } + + if( !owning( ranks ) ) + { + continue; + } + +// EdgeGlbIdx i = offsets.edges.at( ranks ); + i = offsets.edges.at( ranks ); + for( Edge const & edge: edges ) { result.e2n[i] = edge; ++i; @@ -766,19 +812,32 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su // Simple inversion std::map< std::tuple< NodeGlbIdx, NodeGlbIdx >, EdgeGlbIdx > n2e; - for( auto const & [e, n]: result.e2n ) + for( auto const & [e, n]: e2n ) { - n2e[n] = e; + n2e[n] = e; // TODO what about ownership? } std::map< std::vector< NodeGlbIdx >, FaceGlbIdx > n2f; - for( auto const & [ranks, size]: sizes.faces ) + for( auto const & [ranks, faces]: buckets.faces ) { - FaceGlbIdx i{ offsets.faces.at( ranks ) }; - for( Face face: buckets.faces.at( ranks ) ) // Intentional copy for the future `emplace_back`. + FaceGlbIdx i = offsets.faces.at( ranks ); + for( Face const & face: faces ) // TODO hack { n2f[face] = i; - face.emplace_back( face.front() ); + ++i; + } + + if( !owning( ranks ) ) + { + continue; + } + +// FaceGlbIdx i{ offsets.faces.at( ranks ) }; + i = offsets.faces.at( ranks ); + for( Face face: faces ) // Intentional copy for the future `emplace_back`. + { +// n2f[face] = i; + face.emplace_back( face.front() ); // Trick to build the edges. for( std::size_t ii = 0; ii < face.size() - 1; ++ii ) { NodeGlbIdx const & n0 = face[ii], & n1 = face[ii + 1]; @@ -818,6 +877,42 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su return result; } +void assembleAdjacencyMatrix( MeshGraph const & graph, MaxGlbIdcs const & gis, std::size_t const numNodes ) +{ + std::size_t const eo = gis.nodes.get(); + std::size_t const fo = eo + gis.edges.get(); + std::size_t const co = fo + gis.faces.get(); + + std::size_t const n = co + gis.cells.get(); // Total number of entries in the graph. + + std::size_t const numEdges = std::size( graph.e2n ); // TODO Handle to property of the data as well, not the full list! + std::size_t const numFaces = std::size( graph.f2e ); + std::size_t const numCells = std::size( graph.c2f ); + + std::size_t const nnzDiag = numNodes + numEdges + numFaces + numCells; + + std::size_t nnzOffDiag = std::size( graph.e2n ) * 2; + for( auto const & [_, edges]: graph.f2e ) + { + nnzOffDiag += std::size( edges ); + } + for( auto const & [_, faces]: graph.c2f ) + { + nnzOffDiag += std::size( faces ); + } + +// hypre_ParCSRBooleanMatrix * hypre_ParCSRBooleanMatrixCreate( MPI_Comm comm, +// HYPRE_BigInt global_num_rows, +// HYPRE_BigInt global_num_cols, +// HYPRE_BigInt * row_starts, +// HYPRE_BigInt * col_starts, +// HYPRE_Int num_cols_offd, +// HYPRE_Int num_nonzeros_diag, +// HYPRE_Int num_nonzeros_offd ); +// https://github.com/hypre-space/hypre/blob/d475cdfc63ac59ea9a554493a06e4033b8d6fade/src/parcsr_mv/_hypre_parcsr_mv.h#L867 +} + + void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { @@ -877,13 +972,19 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, offsets = deserialize( recvBuffer ); } -// std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; + std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; MpiRank const nextRank = curRank + MpiRank{ 1 }; MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ), offsets.faces.at( { nextRank } ) ); std::cout << "matrixOffsets on rank " << curRank << " -> " << json( matrixOffsets ) << std::endl; - MeshGraph const graph = buildMeshGraph( mesh, buckets, sizes, offsets ); + MeshGraph const graph = buildMeshGraph( mesh, buckets, offsets, curRank ); // TODO change into buildOwnedMeshGraph? + if( curRank == MpiRank{ 1 } ) + { + std::cout << "My graph is " << json( graph ) << std::endl; + } + + assembleAdjacencyMatrix( graph, matrixOffsets, mesh->GetNumberOfPoints() ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From 635be4b4cd8bd1ba798ac8fa4af6bd878525bf7e Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 3 May 2024 18:06:49 -0700 Subject: [PATCH 040/106] Small code simplifications. --- .../mesh/generators/NewGhosting.cpp | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 79d9158cf38..e1969d5f58b 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -786,29 +786,17 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su // The `e2n` is a mapping for all the geometrical entities, not only the one owned like `result.e2n`. // TODO check that it is really useful. std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; - for( auto const & [ranks, edges]: buckets.edges ) { + auto & m = owning( ranks ) ? result.e2n : e2n; EdgeGlbIdx i = offsets.edges.at( ranks ); // TODO hack for( Edge const & edge: edges ) { - e2n[i] = edge; - ++i; - } - - if( !owning( ranks ) ) - { - continue; - } - -// EdgeGlbIdx i = offsets.edges.at( ranks ); - i = offsets.edges.at( ranks ); - for( Edge const & edge: edges ) - { - result.e2n[i] = edge; + m[i] = edge; ++i; } } + e2n.insert( std::cbegin( result.e2n ), std::cend( result.e2n ) ); // Simple inversion std::map< std::tuple< NodeGlbIdx, NodeGlbIdx >, EdgeGlbIdx > n2e; @@ -817,23 +805,14 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su n2e[n] = e; // TODO what about ownership? } - std::map< std::vector< NodeGlbIdx >, FaceGlbIdx > n2f; for( auto const & [ranks, faces]: buckets.faces ) { - FaceGlbIdx i = offsets.faces.at( ranks ); - for( Face const & face: faces ) // TODO hack - { - n2f[face] = i; - ++i; - } - if( !owning( ranks ) ) { continue; } -// FaceGlbIdx i{ offsets.faces.at( ranks ) }; - i = offsets.faces.at( ranks ); + FaceGlbIdx i = offsets.faces.at( ranks ); for( Face face: faces ) // Intentional copy for the future `emplace_back`. { // n2f[face] = i; @@ -850,6 +829,17 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su } } + std::map< std::vector< NodeGlbIdx >, FaceGlbIdx > n2f; + for( auto const & [ranks, faces]: buckets.faces ) + { + FaceGlbIdx i = offsets.faces.at( ranks ); + for( Face const & face: faces ) // TODO hack + { + n2f[face] = i; + ++i; + } + } + vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); vtkIdTypeArray const * globalCellIds = vtkIdTypeArray::FastDownCast( mesh->GetCellData()->GetGlobalIds() ); // TODO do the mapping beforehand for( vtkIdType c = 0; c < mesh->GetNumberOfCells(); ++c ) From 5b82526ba19739d27234b5229bcd3dcfb36e6506 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 8 May 2024 18:00:30 -0700 Subject: [PATCH 041/106] First (a priori) valid assembly of the adjacency graph matrix. Still struggling about the indicator matrix. Also added the owned nodes in the MeshGraph structure. --- host-configs/environment.cmake | 24 +- src/coreComponents/mesh/CMakeLists.txt | 2 +- .../mesh/generators/NewGhosting.cpp | 262 +++++++++++++----- 3 files changed, 212 insertions(+), 76 deletions(-) diff --git a/host-configs/environment.cmake b/host-configs/environment.cmake index c4db8c745a2..fa544b40848 100644 --- a/host-configs/environment.cmake +++ b/host-configs/environment.cmake @@ -28,17 +28,19 @@ endif() if(NOT DEFINED ENABLE_TRILINOS) set(ENABLE_TRILINOS "$ENV{ENABLE_TRILINOS}" CACHE BOOL "" FORCE) endif() -if(ENABLE_TRILINOS) - set(GEOSX_LA_INTERFACE "Trilinos" CACHE STRING "" FORCE) -else() - set(ENABLE_TRILINOS FALSE CACHE BOOL "" FORCE) -endif() - -if( (ENABLE_HYPRE AND ENABLE_TRILINOS) OR (NOT ENABLE_TRILINOS AND NOT ENABLE_HYPRE)) - MESSAGE(SEND_ERROR "Exactly one of ENABLE_HYPRE and ENABLE_TRILINOS must be defined.") - MESSAGE(SEND_ERROR "ENABLE_HYPRE = ${ENABLE_HYPRE}.") - MESSAGE(SEND_ERROR "ENABLE_TRILINOS = ${ENABLE_TRILINOS}.") -endif() +#if(ENABLE_TRILINOS) +# set(GEOSX_LA_INTERFACE "Trilinos" CACHE STRING "" FORCE) +#else() +# set(ENABLE_TRILINOS FALSE CACHE BOOL "" FORCE) +#endif() + +set(ENABLE_TRILINOS TRUE CACHE BOOL "" FORCE) + +#if( (ENABLE_HYPRE AND ENABLE_TRILINOS) OR (NOT ENABLE_TRILINOS AND NOT ENABLE_HYPRE)) +# MESSAGE(SEND_ERROR "Exactly one of ENABLE_HYPRE and ENABLE_TRILINOS must be defined.") +# MESSAGE(SEND_ERROR "ENABLE_HYPRE = ${ENABLE_HYPRE}.") +# MESSAGE(SEND_ERROR "ENABLE_TRILINOS = ${ENABLE_TRILINOS}.") +#endif() MESSAGE(STATUS "GEOSX_LA_INTERFACE = ${GEOSX_LA_INTERFACE}") diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index d0a1fd717ad..f161fe5feb6 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -175,7 +175,7 @@ FetchContent_MakeAvailable(json) #target_link_libraries(foo PRIVATE nlohmann_json::nlohmann_json) -list( APPEND dependencyList NamedType nlohmann_json::nlohmann_json) +list( APPEND dependencyList NamedType nlohmann_json::nlohmann_json trilinos ) if( ENABLE_VTK ) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index e1969d5f58b..d2d905326d4 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -21,7 +21,11 @@ #include "common/MpiWrapper.hpp" #include "common/DataTypes.hpp" -//#include <_hypre_parcsr_mv.h> +#include +#include +#include +#include +#include #include @@ -31,6 +35,7 @@ #include #include + using json = nlohmann::json; #include @@ -43,14 +48,28 @@ namespace geos::ghosting using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::PreIncrementable >; +using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; -using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::PreIncrementable >; +using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; +EdgeGlbIdx operator "" _egi( unsigned long long int i ) +{ + return EdgeGlbIdx{ EdgeGlbIdx::UnderlyingType( i ) }; +} + +FaceGlbIdx operator "" _fgi( unsigned long long int i ) +{ + return FaceGlbIdx{ FaceGlbIdx::UnderlyingType( i ) }; +} + +MpiRank operator "" _mpi( unsigned long long int i ) +{ + return MpiRank{ MpiRank::UnderlyingType( i ) }; +} void to_json( json & j, const MpiRank & v ) @@ -119,7 +138,9 @@ struct Exchange void to_json( json & j, const Exchange & v ) { - j = json{ { "nodes", v.nodes }, { "edges", v.edges }, { "faces", v.faces } }; + j = json{ { "nodes", v.nodes }, + { "edges", v.edges }, + { "faces", v.faces } }; } void from_json( const json & j, @@ -137,6 +158,7 @@ void from_json( const json & j, */ std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) { + // TODO Better handle the boundary information, forgetting about the 3d cells and simply handling the outside shell. auto f = vtkDataSetSurfaceFilter::New(); f->PassThroughCellIdsOn(); f->PassThroughPointIdsOff(); @@ -406,25 +428,25 @@ Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchange } faceBuckets[{ curRank }] = curFaces; - // Checking if neighbors is too wide... // TODO do we care? - std::set< MpiRank > usefulNeighbors; - for( auto const & [ranks, nodes]: nodeBuckets ) - { - if( not nodes.empty() ) - { - usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); - } - } - for( auto const & [ranks, edges]: edgeBuckets ) - { - if( not edges.empty() ) - { - usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); - } - } - std::vector< MpiRank > uselessNeighbors; - std::set_difference( neighbors.cbegin(), neighbors.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); - // TODO... Remove the neighbors? +// // Checking if neighbors is too wide... // TODO do we care here? +// std::set< MpiRank > usefulNeighbors; +// for( auto const & [ranks, nodes]: nodeBuckets ) +// { +// if( not nodes.empty() ) +// { +// usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); +// } +// } +// for( auto const & [ranks, edges]: edgeBuckets ) +// { +// if( not edges.empty() ) +// { +// usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); +// } +// } +// std::vector< MpiRank > uselessNeighbors; +// std::set_difference( neighbors.cbegin(), neighbors.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); +// // TODO... Remove the neighbors? return { nodeBuckets, edgeBuckets, faceBuckets }; } @@ -512,7 +534,8 @@ struct BucketSizes void to_json( json & j, const BucketSizes & v ) { - j = json{ { "edges", v.edges }, { "faces", v.faces } }; + j = json{ { "edges", v.edges }, + { "faces", v.faces } }; } void from_json( const json & j, @@ -531,7 +554,8 @@ struct BucketOffsets void to_json( json & j, const BucketOffsets & v ) { - j = json{ { "edges", v.edges }, { "faces", v.faces } }; + j = json{ { "edges", v.edges }, + { "faces", v.faces } }; } void from_json( const json & j, @@ -614,7 +638,7 @@ std::map< std::set< MpiRank >, GLB_IDX > updateBucketOffsets( std::map< std::set } // Add an extra entry based for the following rank - reducedOffsets.emplace_hint( reducedOffsets.end(), std::set< MpiRank >{ curRank + MpiRank{ 1 } }, nextOffset ); + reducedOffsets.emplace_hint( reducedOffsets.end(), std::set< MpiRank >{ curRank + 1_mpi }, nextOffset ); return reducedOffsets; } @@ -749,6 +773,7 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, struct MeshGraph // TODO add the local <-> global mappings here? { + std::set< NodeGlbIdx > nodes; std::map< CellGlbIdx, std::set< FaceGlbIdx > > c2f; // TODO What about the metadata (e.g. flip the face) std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? @@ -759,7 +784,9 @@ struct MeshGraph // TODO add the local <-> global mappings here? void to_json( json & j, const MeshGraph & v ) // For display { - j = json{ { "c2f", v.f2e }, { "f2e", v.f2e }, { "e2n", v.e2n } }; + j = json{ { "c2f", v.f2e }, + { "f2e", v.f2e }, + { "e2n", v.e2n } }; } @@ -783,6 +810,15 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su return curRank == *std::min_element( std::cbegin( ranks ), std::cend( ranks ) ); }; + for( auto const & [ranks, ns]: buckets.nodes ) + { + if( owning( ranks ) ) + { + result.nodes.insert( std::cbegin( ns ), std::cend( ns ) ); + } + } + + // The `e2n` is a mapping for all the geometrical entities, not only the one owned like `result.e2n`. // TODO check that it is really useful. std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; @@ -851,15 +887,15 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su { vtkCell * face = cell->GetFace( f ); vtkIdList * pids = face->GetPointIds(); - std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); - for( std::size_t i = 0; i < nodes.size(); ++i ) + std::vector< NodeGlbIdx > faceNodes( pids->GetNumberOfIds() ); + for( std::size_t i = 0; i < faceNodes.size(); ++i ) { vtkIdType const lni = face->GetPointId( i ); vtkIdType const gni = globalPtIds->GetValue( lni ); - nodes[i] = NodeGlbIdx{ gni }; + faceNodes[i] = NodeGlbIdx{ gni }; } - std::vector< NodeGlbIdx > const reorderedNodes = reorderFaceNodes( nodes ); - result.c2f[gci].insert( n2f.at( reorderedNodes ) ); + std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes ); + result.c2f[gci].insert( n2f.at( reorderedFaceNodes ) ); // TODO... bool const flipped = ... compare nodes and reorderedNodes. Or ask `reorderFaceNodes` to tell } } @@ -867,39 +903,137 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su return result; } -void assembleAdjacencyMatrix( MeshGraph const & graph, MaxGlbIdcs const & gis, std::size_t const numNodes ) + +void assembleAdjacencyMatrix( MeshGraph const & graph, + MaxGlbIdcs const & gis, + MpiRank curRank ) { - std::size_t const eo = gis.nodes.get(); - std::size_t const fo = eo + gis.edges.get(); - std::size_t const co = fo + gis.faces.get(); + std::size_t const edgeOffset = gis.nodes.get() + 1; + std::size_t const faceOffset = edgeOffset + gis.edges.get() + 1; + std::size_t const cellOffset = faceOffset + gis.faces.get() + 1; - std::size_t const n = co + gis.cells.get(); // Total number of entries in the graph. + std::size_t const n = cellOffset + gis.cells.get() + 1; // Total number of entries in the graph. - std::size_t const numEdges = std::size( graph.e2n ); // TODO Handle to property of the data as well, not the full list! + std::size_t const numNodes = std::size( graph.nodes ); + std::size_t const numEdges = std::size( graph.e2n ); std::size_t const numFaces = std::size( graph.f2e ); std::size_t const numCells = std::size( graph.c2f ); + std::size_t const numOwned = numNodes + numEdges + numFaces + numCells; - std::size_t const nnzDiag = numNodes + numEdges + numFaces + numCells; + std::vector< int > ownedGlbIdcs, numEntriesPerRow; // TODO I couldn't use a vector of `std::size_t` + std::vector< std::vector< int > > indices; + ownedGlbIdcs.reserve( numOwned ); + numEntriesPerRow.reserve( numOwned ); + indices.reserve( numOwned ); - std::size_t nnzOffDiag = std::size( graph.e2n ) * 2; - for( auto const & [_, edges]: graph.f2e ) + for( NodeGlbIdx const & ngi: graph.nodes ) { - nnzOffDiag += std::size( edges ); + auto const i = ngi.get(); + ownedGlbIdcs.emplace_back( i ); + numEntriesPerRow.emplace_back( 1 ); + std::vector< int > const tmp( 1, ownedGlbIdcs.back() ); + indices.emplace_back( tmp ); } - for( auto const & [_, faces]: graph.c2f ) + for( auto const & [egi, nodes]: graph.e2n ) { - nnzOffDiag += std::size( faces ); + auto const i = egi.get() + edgeOffset; + ownedGlbIdcs.emplace_back( i ); + numEntriesPerRow.emplace_back( std::tuple_size_v< decltype( nodes ) > + 1 ); // `+1` comes from the identity + std::vector< int > const tmp{ int( std::get< 0 >( nodes ).get() ), int( std::get< 1 >( nodes ).get() ), ownedGlbIdcs.back() }; + indices.emplace_back( tmp ); } + for( auto const & [fgi, edges]: graph.f2e ) + { + auto const i = fgi.get() + faceOffset; + ownedGlbIdcs.emplace_back( i ); + numEntriesPerRow.emplace_back( std::size( edges ) + 1 ); // `+1` comes from the identity + std::vector< int > tmp; + tmp.reserve( numEntriesPerRow.back() ); + for( EdgeGlbIdx const & egi: edges ) + { + tmp.emplace_back( egi.get() + edgeOffset ); + } + tmp.emplace_back( ownedGlbIdcs.back() ); + indices.emplace_back( tmp ); + } + for( auto const & [cgi, faces]: graph.c2f ) + { + auto const i = cgi.get() + cellOffset; + ownedGlbIdcs.emplace_back( i ); + numEntriesPerRow.emplace_back( std::size( faces ) + 1 ); // `+1` comes from the identity + std::vector< int > tmp; + tmp.reserve( numEntriesPerRow.back() ); + for( FaceGlbIdx const & fgi: faces ) + { + tmp.emplace_back( fgi.get() + faceOffset ); + } + tmp.emplace_back( ownedGlbIdcs.back() ); + indices.emplace_back( tmp ); + } + + GEOS_ASSERT_EQ( numOwned, std::size( ownedGlbIdcs ) ); + GEOS_ASSERT_EQ( numOwned, std::size( numEntriesPerRow ) ); + GEOS_ASSERT_EQ( numOwned, std::size( indices ) ); + for( std::size_t i = 0; i < numOwned; ++i ) + { + GEOS_ASSERT_EQ( indices[i].size(), std::size_t( numEntriesPerRow[i] ) ); + } + +// GEOS_LOG_RANK( "n = " << n ); +// GEOS_LOG_RANK( "ownedGlbIdcs = " << json( ownedGlbIdcs ) ); + + Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); + Epetra_Map const rowMap( n, std::size( ownedGlbIdcs ), ownedGlbIdcs.data(), 0, comm ); -// hypre_ParCSRBooleanMatrix * hypre_ParCSRBooleanMatrixCreate( MPI_Comm comm, -// HYPRE_BigInt global_num_rows, -// HYPRE_BigInt global_num_cols, -// HYPRE_BigInt * row_starts, -// HYPRE_BigInt * col_starts, -// HYPRE_Int num_cols_offd, -// HYPRE_Int num_nonzeros_diag, -// HYPRE_Int num_nonzeros_offd ); -// https://github.com/hypre-space/hypre/blob/d475cdfc63ac59ea9a554493a06e4033b8d6fade/src/parcsr_mv/_hypre_parcsr_mv.h#L867 + Epetra_CrsMatrix adj( Epetra_DataAccess::Copy, rowMap, numEntriesPerRow.data(), true ); + + for( std::size_t i = 0; i < numOwned; ++i ) + { + std::vector< int > const & rowIndices = indices[i]; + std::vector< double > const rowValues( std::size( rowIndices ), 1. ); + GEOS_ASSERT_EQ( std::size( rowIndices ), std::size_t( numEntriesPerRow[i] ) ); + adj.InsertGlobalValues( ownedGlbIdcs[i], std::size( rowIndices ), rowValues.data(), rowIndices.data() ); + } + + adj.FillComplete(); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/adj.mat", adj ); + + // Now let's build the domain indicator matrix. + // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. + Epetra_Map const rowMap2( MpiWrapper::commSize(), 1, 0, comm ); + Epetra_Map const colMap2( int( n ), numOwned, ownedGlbIdcs.data(), 0, comm ); +// rowMap2.Print( std::cout ); +// colMap2.Print( std::cout ); +// Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap2, std::size( ownedGlbIdcs ), true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? +// Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap2, colMap2, std::size( ownedGlbIdcs ), true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? + Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap2, colMap2, numOwned, true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? + std::vector< double > const one( numOwned, 1. ); + indicator.InsertGlobalValues( curRank.get(), numOwned, one.data(), ownedGlbIdcs.data() ); + indicator.FillComplete(); + GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); + GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/indicator.mat", indicator ); + +// // Now let's build the domain indicator matrix. +// // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. +// Epetra_Map const colMap( MpiWrapper::commSize(), 1, 0, Epetra_MpiComm( MPI_COMM_GEOSX ) ); +// colMap.Print(std::cout); +// // TODO Filling the transposition is surely easier... +// Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap, colMap, 1, true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? +// std::vector< double > const one( 1, 1. ); +// std::vector< int > const rank( 1, curRank.get() ); +// for( std::size_t i = 0; i < numOwned; ++i ) +// { +//// indicator.InsertMyValues( i, 1, one.data(), rank.data() ); // TODO WRONG! +// indicator.InsertGlobalValues( ownedGlbIdcs[i], 1, one.data(), rank.data() ); // TODO WRONG! +// } +// indicator.FillComplete(); +// GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); +// GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); +// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/indicator.mat", indicator ); + + Epetra_CrsMatrix result( Epetra_DataAccess::Copy, rowMap, numEntriesPerRow.data() ); // TODO bad estimation + EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result ); } @@ -931,12 +1065,12 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, BucketSizes const sizes = getBucketSize( buckets ); BucketOffsets offsets; - if( curRank == MpiRank{ 0 } ) + if( curRank == 0_mpi ) { // The `MPI_Scan` process will not call the reduction operator for rank 0. // So we need to reduce ourselves for ourselves. - offsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes.edges, { { { MpiRank{ 0 }, }, EdgeGlbIdx { 0 } } }, curRank ); - offsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes.faces, { { { MpiRank{ 0 }, }, FaceGlbIdx { 0 } } }, curRank ); + offsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes.edges, { { { 0_mpi, }, 0_egi } }, curRank ); + offsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes.faces, { { { 0_mpi, }, 0_fgi } }, curRank ); // Still we need to send this reduction to the following rank, by copying to it to the send buffer. std::vector< std::uint8_t > const bytes = serialize( offsets ); std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); @@ -957,24 +1091,24 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, MPI_Scan( sendBuffer.data(), recvBuffer.data(), maxBufferSize, MPI_BYTE, op, MPI_COMM_WORLD ); - if( curRank != MpiRank{ 0 } ) + if( curRank != 0_mpi ) { offsets = deserialize( recvBuffer ); } std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; - MpiRank const nextRank = curRank + MpiRank{ 1 }; - MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ), offsets.faces.at( { nextRank } ) ); + MpiRank const nextRank = curRank + 1_mpi; + MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ) - 1_egi, offsets.faces.at( { nextRank } ) - 1_fgi ); std::cout << "matrixOffsets on rank " << curRank << " -> " << json( matrixOffsets ) << std::endl; MeshGraph const graph = buildMeshGraph( mesh, buckets, offsets, curRank ); // TODO change into buildOwnedMeshGraph? - if( curRank == MpiRank{ 1 } ) - { - std::cout << "My graph is " << json( graph ) << std::endl; - } +// if( curRank == MpiRank{ 1 } ) +// { +// std::cout << "My graph is " << json( graph ) << std::endl; +// } - assembleAdjacencyMatrix( graph, matrixOffsets, mesh->GetNumberOfPoints() ); + assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From 4c214a280686df57431eb3f942ba60a9a5ac0cb6 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 9 May 2024 14:58:42 -0700 Subject: [PATCH 042/106] Handle rectanguler trilinos matrix using domain and range maps. --- scripts/rectangular_trilinos.cpp | 54 +++++++++++++++++++ .../mesh/generators/NewGhosting.cpp | 33 +++--------- 2 files changed, 61 insertions(+), 26 deletions(-) create mode 100644 scripts/rectangular_trilinos.cpp diff --git a/scripts/rectangular_trilinos.cpp b/scripts/rectangular_trilinos.cpp new file mode 100644 index 00000000000..30b14ffdd49 --- /dev/null +++ b/scripts/rectangular_trilinos.cpp @@ -0,0 +1,54 @@ +/* + * Compile: mpicxx -std=c++20 -I /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/include/ rectangular_trilinos.cpp -L /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/lib -lepetra -Wl,-rpath /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/lib -o /tmp/rectangular_trilinos + * Launch: mpirun -n 3 /tmp/rectangular_trilinos + */ + +#include +#include +#include +#include +#include + +#include + +#include + +int main( int argc, + char ** argv ) +{ + int rank, size; + MPI_Init( &argc, &argv ); + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + MPI_Comm_size( MPI_COMM_WORLD, &size ); + + const int numGlbCols = 4; + const int numGlbRows = size; + + Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_WORLD ); + std::vector< int > const ownedGlbRows{ rank }; + + Epetra_Map const rowMap( numGlbRows, 1, ownedGlbRows.data(), 0, comm ); + Epetra_CrsMatrix m( Epetra_DataAccess::Copy, rowMap, numGlbCols, true ); + + std::vector< double > const value{ 1 + 10. * rank, 2 + 10. * rank, 3 + 10. * rank, 4 + 10. * rank }; + m.InsertGlobalValues( rank, numGlbCols, value.data(), std::vector< int >{ 3, 2, 1, 0 }.data() ); + + Epetra_Map const domainMap( numGlbCols, 0, comm ); + Epetra_Map const rangeMap( numGlbRows, 0, comm ); + +// m.FillComplete(); + m.FillComplete( domainMap, rangeMap ); + +// m.ColMap().Print(std::cout); + + m.DomainMap().Print( std::cout ); + + std::cout << "glb(rows, cols) = " << m.NumGlobalRows64() << " " << m.NumGlobalCols64() << std::endl; + std::cout << "loc(rows, cols) = " << m.NumMyRows() << " " << m.NumMyCols() << std::endl; + +// m.Print( std::cout ); + + MPI_Finalize(); + + return 0; +} \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index d2d905326d4..17d8ce42cbe 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1000,38 +1000,19 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // Now let's build the domain indicator matrix. // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. - Epetra_Map const rowMap2( MpiWrapper::commSize(), 1, 0, comm ); - Epetra_Map const colMap2( int( n ), numOwned, ownedGlbIdcs.data(), 0, comm ); -// rowMap2.Print( std::cout ); -// colMap2.Print( std::cout ); -// Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap2, std::size( ownedGlbIdcs ), true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? -// Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap2, colMap2, std::size( ownedGlbIdcs ), true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? - Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap2, colMap2, numOwned, true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? + Epetra_Map const indicatorRowMap( MpiWrapper::commSize(), 1, &curRank.get(), 0, comm ); + Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, indicatorRowMap, numOwned, true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? std::vector< double > const one( numOwned, 1. ); indicator.InsertGlobalValues( curRank.get(), numOwned, one.data(), ownedGlbIdcs.data() ); - indicator.FillComplete(); + Epetra_Map const domainMap( int( n ), 0, comm ); // Columns + Epetra_Map const rangeMap( MpiWrapper::commSize(), 0, comm ); // Rows + indicator.FillComplete( domainMap, rangeMap ); + GEOS_LOG_RANK( "indicator.NumMyRows() = " << indicator.NumMyRows() ); + GEOS_LOG_RANK( "indicator.NumMyCols() = " << indicator.NumMyCols() ); GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/indicator.mat", indicator ); -// // Now let's build the domain indicator matrix. -// // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. -// Epetra_Map const colMap( MpiWrapper::commSize(), 1, 0, Epetra_MpiComm( MPI_COMM_GEOSX ) ); -// colMap.Print(std::cout); -// // TODO Filling the transposition is surely easier... -// Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap, colMap, 1, true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? -// std::vector< double > const one( 1, 1. ); -// std::vector< int > const rank( 1, curRank.get() ); -// for( std::size_t i = 0; i < numOwned; ++i ) -// { -//// indicator.InsertMyValues( i, 1, one.data(), rank.data() ); // TODO WRONG! -// indicator.InsertGlobalValues( ownedGlbIdcs[i], 1, one.data(), rank.data() ); // TODO WRONG! -// } -// indicator.FillComplete(); -// GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); -// GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); -// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/indicator.mat", indicator ); - Epetra_CrsMatrix result( Epetra_DataAccess::Copy, rowMap, numEntriesPerRow.data() ); // TODO bad estimation EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result ); } From 401c5bc7cc0e53d972b0ed7b7f34997b0b8044f9 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 10 May 2024 09:34:27 -0700 Subject: [PATCH 043/106] First adjacency matrix multiplications that go through. Save commit before reorgs. --- .../mesh/generators/NewGhosting.cpp | 104 ++++++++++++++++-- 1 file changed, 95 insertions(+), 9 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 17d8ce42cbe..0f84dd9b04b 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -26,6 +26,8 @@ #include #include #include +#include + #include @@ -996,7 +998,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, } adj.FillComplete(); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/adj.mat", adj ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/adj.mat", adj ); // Now let's build the domain indicator matrix. // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. @@ -1007,14 +1009,98 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_Map const domainMap( int( n ), 0, comm ); // Columns Epetra_Map const rangeMap( MpiWrapper::commSize(), 0, comm ); // Rows indicator.FillComplete( domainMap, rangeMap ); - GEOS_LOG_RANK( "indicator.NumMyRows() = " << indicator.NumMyRows() ); - GEOS_LOG_RANK( "indicator.NumMyCols() = " << indicator.NumMyCols() ); - GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); - GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/indicator.mat", indicator ); - - Epetra_CrsMatrix result( Epetra_DataAccess::Copy, rowMap, numEntriesPerRow.data() ); // TODO bad estimation - EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result ); + if( curRank == 0_mpi ) + { + GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); + GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); + } + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); + + int NN( MpiWrapper::commSize() ); + GEOS_LOG_RANK("A"); + auto multiply = [&]()-> Epetra_CrsMatrix + { + // Upward (n -> e -> f -> c) + + Epetra_CrsMatrix result0( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result0, false ); + result0.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result0 ); + + Epetra_CrsMatrix result1( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, result0, false, result1, false ); + result1.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result1 ); + + Epetra_CrsMatrix result2( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, result1, false, result2, false ); + result2.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result2 ); + + // Unneeded step + Epetra_CrsMatrix result3( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, result2, false, result3, false ); + result3.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-3.mat", result3 ); + + // Downward (c -> f -> e -> n) + + Epetra_CrsMatrix result4( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result3, false, result4, false ); + result4.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result4 ); + + Epetra_CrsMatrix result5( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result4, false, result5, false ); + result5.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result5 ); + + Epetra_CrsMatrix result6( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result5, false, result6, false ); + result6.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result6 ); + + // Unneeded step + Epetra_CrsMatrix result7( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result6, false, result7, false ); + result7.FillComplete( rangeMap, domainMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-7.mat", result7 ); + return result7; + }; + Epetra_CrsMatrix ghosted( multiply() ); + ghosted.FillComplete( rangeMap, domainMap ); + Epetra_RowMatrixTransposer transposer( &ghosted ); + Epetra_CrsMatrix tGhosted( Epetra_DataAccess::Copy, indicatorRowMap, int( n ), false ); + Epetra_CrsMatrix * ptGhosted = &tGhosted; + transposer.CreateTranspose( true, ptGhosted ); + tGhosted.FillComplete( domainMap, rangeMap ); + GEOS_LOG_RANK( "tGhosted.NumGlobalCols() = " << tGhosted.NumGlobalCols() ); + GEOS_LOG_RANK( "tGhosted.NumGlobalRows() = " << tGhosted.NumGlobalRows() ); + + // My test says that `tGhosted` is filled here. + int extracted = 0; + std::vector< double > extractedValues(n); +// extractedValues.reserve( n ); + std::vector< int > extractedIndices(n); +// extractedIndices.reserve( n ); + tGhosted.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); + GEOS_LOG_RANK( "extracted = " << extracted ); + { + std::vector< int > cells; + for( int i = 0; i < extracted; ++i ) + { + int const & index = extractedIndices[i]; + double const & val = extractedValues[i]; + if( val > 0 and index > int(cellOffset) ) + { + cells.push_back( index - cellOffset ); + } + } + GEOS_LOG_RANK( "ghost cells = " << json( cells ) ); + } + +// ghosted.Print( std::cout ); + GEOS_LOG_RANK("B"); } From 0ab79a1ca8624392a76baa836fbaa31109f88de6 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 10 May 2024 10:16:19 -0700 Subject: [PATCH 044/106] Little renamings for the domain/range maps --- .../mesh/generators/NewGhosting.cpp | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 0f84dd9b04b..ada95f1a986 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -985,7 +985,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // GEOS_LOG_RANK( "ownedGlbIdcs = " << json( ownedGlbIdcs ) ); Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); - Epetra_Map const rowMap( n, std::size( ownedGlbIdcs ), ownedGlbIdcs.data(), 0, comm ); + Epetra_Map const rowMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); Epetra_CrsMatrix adj( Epetra_DataAccess::Copy, rowMap, numEntriesPerRow.data(), true ); @@ -1002,13 +1002,17 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // Now let's build the domain indicator matrix. // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. - Epetra_Map const indicatorRowMap( MpiWrapper::commSize(), 1, &curRank.get(), 0, comm ); - Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, indicatorRowMap, numOwned, true ); // Maybe `rowMap` is bad... Or `Epetra_CrsMatrix` wrongly assumes it's square? - std::vector< double > const one( numOwned, 1. ); - indicator.InsertGlobalValues( curRank.get(), numOwned, one.data(), ownedGlbIdcs.data() ); - Epetra_Map const domainMap( int( n ), 0, comm ); // Columns - Epetra_Map const rangeMap( MpiWrapper::commSize(), 0, comm ); // Rows - indicator.FillComplete( domainMap, rangeMap ); + Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap, 1, true ); + constexpr double one{ 1. }; + for( int const & gi: ownedGlbIdcs ) + { + indicator.InsertGlobalValues( gi, 1, &one, &curRank.get() ); + + } + Epetra_Map const mpiMap( MpiWrapper::commSize(), 0, comm ); // Rows + Epetra_Map const graphNodeMap( int( n ), 0, comm ); // Columns + indicator.FillComplete( mpiMap, graphNodeMap ); + if( curRank == 0_mpi ) { GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); @@ -1023,65 +1027,68 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // Upward (n -> e -> f -> c) Epetra_CrsMatrix result0( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result0, false ); - result0.FillComplete( rangeMap, domainMap ); + EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, false, result0, false ); + result0.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result0 ); Epetra_CrsMatrix result1( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result0, false, result1, false ); - result1.FillComplete( rangeMap, domainMap ); + result1.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result1 ); Epetra_CrsMatrix result2( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result1, false, result2, false ); - result2.FillComplete( rangeMap, domainMap ); + result2.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result2 ); // Unneeded step Epetra_CrsMatrix result3( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result2, false, result3, false ); - result3.FillComplete( rangeMap, domainMap ); + result3.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-3.mat", result3 ); // Downward (c -> f -> e -> n) Epetra_CrsMatrix result4( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result3, false, result4, false ); - result4.FillComplete( rangeMap, domainMap ); + result4.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result4 ); Epetra_CrsMatrix result5( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result4, false, result5, false ); - result5.FillComplete( rangeMap, domainMap ); + result5.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result5 ); Epetra_CrsMatrix result6( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result5, false, result6, false ); - result6.FillComplete( rangeMap, domainMap ); + result6.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result6 ); // Unneeded step Epetra_CrsMatrix result7( Epetra_DataAccess::Copy, rowMap, NN, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result6, false, result7, false ); - result7.FillComplete( rangeMap, domainMap ); + result7.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-7.mat", result7 ); return result7; }; + Epetra_CrsMatrix ghosted( multiply() ); - ghosted.FillComplete( rangeMap, domainMap ); + ghosted.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); + Epetra_RowMatrixTransposer transposer( &ghosted ); - Epetra_CrsMatrix tGhosted( Epetra_DataAccess::Copy, indicatorRowMap, int( n ), false ); + Epetra_CrsMatrix tGhosted( Epetra_DataAccess::Copy, mpiMap, int( n ), false ); Epetra_CrsMatrix * ptGhosted = &tGhosted; transposer.CreateTranspose( true, ptGhosted ); - tGhosted.FillComplete( domainMap, rangeMap ); + tGhosted.FillComplete( graphNodeMap, mpiMap ); GEOS_LOG_RANK( "tGhosted.NumGlobalCols() = " << tGhosted.NumGlobalCols() ); GEOS_LOG_RANK( "tGhosted.NumGlobalRows() = " << tGhosted.NumGlobalRows() ); // My test says that `tGhosted` is filled here. int extracted = 0; - std::vector< double > extractedValues(n); + std::vector< double > extractedValues( n ); // extractedValues.reserve( n ); - std::vector< int > extractedIndices(n); + std::vector< int > extractedIndices( n ); // extractedIndices.reserve( n ); tGhosted.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); GEOS_LOG_RANK( "extracted = " << extracted ); @@ -1099,7 +1106,6 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_LOG_RANK( "ghost cells = " << json( cells ) ); } -// ghosted.Print( std::cout ); GEOS_LOG_RANK("B"); } From 638a659006bf937f322b73bffe47c1a420fbf0d0 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 10 May 2024 10:34:09 -0700 Subject: [PATCH 045/106] Handling the transposition of the ghosting matrix + extracting the row. --- .../mesh/generators/NewGhosting.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index ada95f1a986..489552cf061 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1076,21 +1076,20 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, ghosted.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); + // Transposing in order to get easy access to the row... Epetra_RowMatrixTransposer transposer( &ghosted ); - Epetra_CrsMatrix tGhosted( Epetra_DataAccess::Copy, mpiMap, int( n ), false ); - Epetra_CrsMatrix * ptGhosted = &tGhosted; + Epetra_CrsMatrix * ptGhosted = nullptr; // The transposer returns a pointer we must handle ourselves. transposer.CreateTranspose( true, ptGhosted ); - tGhosted.FillComplete( graphNodeMap, mpiMap ); - GEOS_LOG_RANK( "tGhosted.NumGlobalCols() = " << tGhosted.NumGlobalCols() ); - GEOS_LOG_RANK( "tGhosted.NumGlobalRows() = " << tGhosted.NumGlobalRows() ); + GEOS_LOG_RANK( "ptGhosted->NumGlobalCols() = " << ptGhosted->NumGlobalCols() ); + GEOS_LOG_RANK( "ptGhosted->NumGlobalRows() = " << ptGhosted->NumGlobalRows() ); // My test says that `tGhosted` is filled here. int extracted = 0; std::vector< double > extractedValues( n ); -// extractedValues.reserve( n ); std::vector< int > extractedIndices( n ); -// extractedIndices.reserve( n ); - tGhosted.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); + ptGhosted->ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); + extractedValues.resize( extracted ); + extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); { std::vector< int > cells; @@ -1107,6 +1106,8 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, } GEOS_LOG_RANK("B"); + + delete ptGhosted; } From 5058c298fe612671b821794400fed54c608d7223 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 10 May 2024 15:40:14 -0700 Subject: [PATCH 046/106] This version managegs appropriately the quantities that are owned and the quantities that are there nevertheless. --- .../mesh/generators/NewGhosting.cpp | 125 ++++++++++++------ 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 489552cf061..51125facecd 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -43,6 +43,18 @@ using json = nlohmann::json; #include #include +namespace geos +{ + +template< typename OUTPUT, typename INPUT > +inline LVARRAY_HOST_DEVICE +OUTPUT intConv( INPUT input ) +{ + return LvArray::integerConversion< OUTPUT >( input ); +} + +} // end of namespace + namespace geos::ghosting { @@ -775,10 +787,13 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, struct MeshGraph // TODO add the local <-> global mappings here? { - std::set< NodeGlbIdx > nodes; std::map< CellGlbIdx, std::set< FaceGlbIdx > > c2f; // TODO What about the metadata (e.g. flip the face) std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? + std::set< NodeGlbIdx > nodes; + std::set< FaceGlbIdx > otherFaces; // Faces that are there but not owned. + std::set< EdgeGlbIdx > otherEdges; // Edges that are there but not owned. + std::set< NodeGlbIdx > otherNodes; // Nodes that are there but not owned. // TODO add the nodes here? // TODO add all types of connections here? How? }; @@ -807,30 +822,32 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su { MeshGraph result; - auto const owning = [&curRank]( std::set< MpiRank > const & ranks ) -> bool + auto const isCurrentRankOwning = [&curRank]( std::set< MpiRank > const & ranks ) -> bool { return curRank == *std::min_element( std::cbegin( ranks ), std::cend( ranks ) ); }; for( auto const & [ranks, ns]: buckets.nodes ) { - if( owning( ranks ) ) - { - result.nodes.insert( std::cbegin( ns ), std::cend( ns ) ); - } + std::set< NodeGlbIdx > & nodes = isCurrentRankOwning( ranks ) ? result.nodes : result.otherNodes; + nodes.insert( std::cbegin( ns ), std::cend( ns ) ); } - // The `e2n` is a mapping for all the geometrical entities, not only the one owned like `result.e2n`. // TODO check that it is really useful. std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; for( auto const & [ranks, edges]: buckets.edges ) { - auto & m = owning( ranks ) ? result.e2n : e2n; + bool const isOwning = isCurrentRankOwning( ranks ); + auto & m = isOwning ? result.e2n : e2n; EdgeGlbIdx i = offsets.edges.at( ranks ); // TODO hack for( Edge const & edge: edges ) { m[i] = edge; + if( not isOwning ) + { + result.otherEdges.insert( i ); // TODO use the keys of e2n instead? + } ++i; } } @@ -845,25 +862,32 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su for( auto const & [ranks, faces]: buckets.faces ) { - if( !owning( ranks ) ) + bool const isOwning = isCurrentRankOwning( ranks ); + + if( isOwning ) { - continue; + FaceGlbIdx i = offsets.faces.at( ranks ); + for( Face face: faces ) // Intentional copy for the future `emplace_back`. + { + face.emplace_back( face.front() ); // Trick to build the edges. + for( std::size_t ii = 0; ii < face.size() - 1; ++ii ) + { + NodeGlbIdx const & n0 = face[ii], & n1 = face[ii + 1]; + std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); + std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); + result.f2e[i].insert( n2e.at( p1 ) ); + bool const flipped = p0 != p1; // TODO store somewhere. + } + ++i; + } } - - FaceGlbIdx i = offsets.faces.at( ranks ); - for( Face face: faces ) // Intentional copy for the future `emplace_back`. + else { -// n2f[face] = i; - face.emplace_back( face.front() ); // Trick to build the edges. - for( std::size_t ii = 0; ii < face.size() - 1; ++ii ) + FaceGlbIdx const size = FaceGlbIdx{ intConv< FaceGlbIdx::UnderlyingType >( std::size( faces ) ) }; + for( FaceGlbIdx ii = offsets.faces.at( ranks ); ii < size; ++ii ) { - NodeGlbIdx const & n0 = face[ii], & n1 = face[ii + 1]; - std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); - std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); - result.f2e[i].insert( n2e.at( p1 ) ); - bool const flipped = p0 != p1; // TODO store somewhere. + result.otherFaces.insert( ii ); // TODO insert iota } - ++i; } } @@ -916,11 +940,16 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, std::size_t const n = cellOffset + gis.cells.get() + 1; // Total number of entries in the graph. - std::size_t const numNodes = std::size( graph.nodes ); - std::size_t const numEdges = std::size( graph.e2n ); - std::size_t const numFaces = std::size( graph.f2e ); - std::size_t const numCells = std::size( graph.c2f ); - std::size_t const numOwned = numNodes + numEdges + numFaces + numCells; + std::size_t const numOwnedNodes = std::size( graph.nodes ); + std::size_t const numOwnedEdges = std::size( graph.e2n ); + std::size_t const numOwnedFaces = std::size( graph.f2e ); + std::size_t const numOwnedCells = std::size( graph.c2f ); + std::size_t const numOwned = numOwnedNodes + numOwnedEdges + numOwnedFaces + numOwnedCells; + + std::size_t const numOtherNodes = std::size( graph.otherNodes ); + std::size_t const numOtherEdges = std::size( graph.otherEdges ); + std::size_t const numOtherFaces = std::size( graph.otherFaces ); + std::size_t const numOther = numOtherNodes + numOtherEdges + numOtherFaces; std::vector< int > ownedGlbIdcs, numEntriesPerRow; // TODO I couldn't use a vector of `std::size_t` std::vector< std::vector< int > > indices; @@ -928,6 +957,22 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, numEntriesPerRow.reserve( numOwned ); indices.reserve( numOwned ); + std::vector< int > otherGlbIdcs; // TODO I couldn't use a vector of `std::size_t` + otherGlbIdcs.reserve( numOther ); + for( NodeGlbIdx const & ngi: graph.otherNodes ) + { + otherGlbIdcs.emplace_back( ngi.get() ); + } + for( EdgeGlbIdx const & egi: graph.otherEdges ) + { + otherGlbIdcs.emplace_back( egi.get() + edgeOffset ); + } + for( FaceGlbIdx const & fgi: graph.otherFaces ) + { + otherGlbIdcs.emplace_back( fgi.get() + faceOffset ); + } + GEOS_ASSERT_EQ( numOther, std::size( otherGlbIdcs ) ); + for( NodeGlbIdx const & ngi: graph.nodes ) { auto const i = ngi.get(); @@ -1002,16 +1047,15 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // Now let's build the domain indicator matrix. // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. - Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, rowMap, 1, true ); - constexpr double one{ 1. }; - for( int const & gi: ownedGlbIdcs ) - { - indicator.InsertGlobalValues( gi, 1, &one, &curRank.get() ); - - } Epetra_Map const mpiMap( MpiWrapper::commSize(), 0, comm ); // Rows + Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, mpiMap, numOwned + numOther, true ); + + std::vector< double > const ones( n, 1. ); + indicator.InsertGlobalValues( curRank.get(), numOwned, ones.data(), ownedGlbIdcs.data() ); + indicator.InsertGlobalValues( curRank.get(), numOther, ones.data(), otherGlbIdcs.data() ); + Epetra_Map const graphNodeMap( int( n ), 0, comm ); // Columns - indicator.FillComplete( mpiMap, graphNodeMap ); + indicator.FillComplete( graphNodeMap, mpiMap ); if( curRank == 0_mpi ) { @@ -1027,7 +1071,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // Upward (n -> e -> f -> c) Epetra_CrsMatrix result0( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, false, result0, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result0, false ); result0.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result0 ); @@ -1092,17 +1136,18 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); { - std::vector< int > cells; + std::vector< int > interest; for( int i = 0; i < extracted; ++i ) { int const & index = extractedIndices[i]; double const & val = extractedValues[i]; - if( val > 0 and index > int(cellOffset) ) + if( val > 0 and index < int( edgeOffset ) ) { - cells.push_back( index - cellOffset ); +// interest.push_back( index - cellOffset ); + interest.push_back( index ); } } - GEOS_LOG_RANK( "ghost cells = " << json( cells ) ); + GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); } GEOS_LOG_RANK("B"); From 43adb71d7dc635f9ecb841d976e9757194b5e9c5 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 10 May 2024 16:32:39 -0700 Subject: [PATCH 047/106] Removing one step in the adjacency multiplication process. --- .../mesh/generators/NewGhosting.cpp | 71 ++++++++----------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 51125facecd..957bbad65d5 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1070,50 +1070,39 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, { // Upward (n -> e -> f -> c) - Epetra_CrsMatrix result0( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result0, false ); - result0.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result0 ); - - Epetra_CrsMatrix result1( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, result0, false, result1, false ); - result1.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result1 ); - - Epetra_CrsMatrix result2( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, result1, false, result2, false ); - result2.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result2 ); - - // Unneeded step - Epetra_CrsMatrix result3( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, result2, false, result3, false ); - result3.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-3.mat", result3 ); + Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result_u0_0, false ); + result_u0_0.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); + + Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_0, false, result_u0_1, false ); + result_u0_1.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); + + Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_1, false, result_u0_2, false ); + result_u0_2.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); // Downward (c -> f -> e -> n) - Epetra_CrsMatrix result4( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result3, false, result4, false ); - result4.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result4 ); - - Epetra_CrsMatrix result5( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result4, false, result5, false ); - result5.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result5 ); - - Epetra_CrsMatrix result6( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result5, false, result6, false ); - result6.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result6 ); - - // Unneeded step - Epetra_CrsMatrix result7( Epetra_DataAccess::Copy, rowMap, NN, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result6, false, result7, false ); - result7.FillComplete( mpiMap, graphNodeMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-7.mat", result7 ); - return result7; + Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result_u0_2, false, result_d0_0, false ); + result_d0_0.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); + + Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_0, false, result_d0_1, false ); + result_d0_1.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); + + Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, rowMap, NN, false ); + EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_1, false, result_d0_2, false ); + result_d0_2.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); + + return result_d0_2; }; Epetra_CrsMatrix ghosted( multiply() ); From f88b927c4ef8323cbc00bccd746a9310197d0aac Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 14 May 2024 14:52:09 -0700 Subject: [PATCH 048/106] Print cells instead. --- src/coreComponents/mesh/generators/NewGhosting.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 957bbad65d5..cb5b4f0615e 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -47,7 +47,7 @@ namespace geos { template< typename OUTPUT, typename INPUT > -inline LVARRAY_HOST_DEVICE +inline GEOS_HOST_DEVICE OUTPUT intConv( INPUT input ) { return LvArray::integerConversion< OUTPUT >( input ); @@ -1130,11 +1130,14 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, { int const & index = extractedIndices[i]; double const & val = extractedValues[i]; - if( val > 0 and index < int( edgeOffset ) ) + if( val > 0 and index >= int( cellOffset ) ) { -// interest.push_back( index - cellOffset ); - interest.push_back( index ); + interest.push_back( index - cellOffset ); } +// if( val > 0 and index < int( edgeOffset ) ) +// { +// interest.push_back( index ); +// } } GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); } From 96b2eb28d28fb70b56158321d87e6ff1f729d55d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 14 May 2024 16:37:34 -0700 Subject: [PATCH 049/106] Create an include folder for the public interface --- src/coreComponents/mesh/CMakeLists.txt | 4 +- .../mesh/ElementRegionManager.cpp | 2 +- src/coreComponents/mesh/PerforationData.hpp | 2 +- src/coreComponents/mesh/WellElementRegion.hpp | 2 +- .../mesh/WellElementSubRegion.hpp | 2 +- .../mesh/generators/CellBlockManager.hpp | 2 +- .../mesh/generators/CellBlockManagerABC.hpp | 2 +- .../mesh/generators/LineBlock.hpp | 2 +- .../mesh/generators/MeshBlockMappings.cpp | 142 ------------------ .../mesh/generators/MeshBlockMappings.hpp | 103 ------------- .../mesh/generators/WellGeneratorBase.cpp | 2 +- .../mesh/generators/include/CellMgr.hpp | 69 +++++++++ .../mesh/generators/include/EdgeMgr.hpp | 49 ++++++ .../mesh/generators/include/FaceMgr.hpp | 57 +++++++ .../generators/{ => include}/LineBlockABC.hpp | 2 +- .../mesh/generators/include/MeshMappings.hpp | 92 ++++++++++++ .../mesh/generators/include/NodeMgr.hpp | 76 ++++++++++ 17 files changed, 353 insertions(+), 257 deletions(-) delete mode 100644 src/coreComponents/mesh/generators/MeshBlockMappings.cpp delete mode 100644 src/coreComponents/mesh/generators/MeshBlockMappings.hpp create mode 100644 src/coreComponents/mesh/generators/include/CellMgr.hpp create mode 100644 src/coreComponents/mesh/generators/include/EdgeMgr.hpp create mode 100644 src/coreComponents/mesh/generators/include/FaceMgr.hpp rename src/coreComponents/mesh/generators/{ => include}/LineBlockABC.hpp (98%) create mode 100644 src/coreComponents/mesh/generators/include/MeshMappings.hpp create mode 100644 src/coreComponents/mesh/generators/include/NodeMgr.hpp diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index f161fe5feb6..59bdb77afd9 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -47,14 +47,13 @@ set( mesh_headers generators/CellBlockManagerABC.hpp generators/CellBlockUtilities.hpp generators/LineBlock.hpp - generators/LineBlockABC.hpp + generators/include/LineBlockABC.hpp generators/ExternalMeshGeneratorBase.hpp generators/FaceBlock.hpp generators/FaceBlockABC.hpp generators/InternalMeshGenerator.hpp generators/InternalWellGenerator.hpp generators/InternalWellboreGenerator.hpp - generators/MeshBlockMappings.hpp generators/MeshGeneratorBase.hpp generators/ParMETISInterface.hpp generators/ParticleMeshGenerator.hpp @@ -128,7 +127,6 @@ set( mesh_sources generators/InternalMeshGenerator.cpp generators/InternalWellGenerator.cpp generators/InternalWellboreGenerator.cpp - generators/MeshBlockMappings.cpp generators/MeshGeneratorBase.cpp generators/ParMETISInterface.cpp generators/ParticleMeshGenerator.cpp diff --git a/src/coreComponents/mesh/ElementRegionManager.cpp b/src/coreComponents/mesh/ElementRegionManager.cpp index b967a2b9e76..7b242bb5a22 100644 --- a/src/coreComponents/mesh/ElementRegionManager.cpp +++ b/src/coreComponents/mesh/ElementRegionManager.cpp @@ -26,7 +26,7 @@ #include "mesh/MeshLevel.hpp" #include "mesh/utilities/MeshMapUtilities.hpp" #include "schema/schemaUtilities.hpp" -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/PerforationData.hpp b/src/coreComponents/mesh/PerforationData.hpp index 0c3a4828e03..9bc8002fd9f 100644 --- a/src/coreComponents/mesh/PerforationData.hpp +++ b/src/coreComponents/mesh/PerforationData.hpp @@ -22,7 +22,7 @@ #include "dataRepository/Group.hpp" #include "mesh/ObjectManagerBase.hpp" #include "mesh/ToElementRelation.hpp" -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/WellElementRegion.hpp b/src/coreComponents/mesh/WellElementRegion.hpp index c3416030931..41416b4f16f 100644 --- a/src/coreComponents/mesh/WellElementRegion.hpp +++ b/src/coreComponents/mesh/WellElementRegion.hpp @@ -21,7 +21,7 @@ #define GEOS_MESH_WELLELEMENTREGION_HPP_ #include "mesh/ElementRegionBase.hpp" -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/WellElementSubRegion.hpp b/src/coreComponents/mesh/WellElementSubRegion.hpp index 9155ed79139..cc45fa287d4 100644 --- a/src/coreComponents/mesh/WellElementSubRegion.hpp +++ b/src/coreComponents/mesh/WellElementSubRegion.hpp @@ -18,7 +18,7 @@ #include "mesh/ElementSubRegionBase.hpp" #include "mesh/InterObjectRelation.hpp" #include "mesh/PerforationData.hpp" -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/generators/CellBlockManager.hpp b/src/coreComponents/mesh/generators/CellBlockManager.hpp index 517e7f61f0f..5370b220c79 100644 --- a/src/coreComponents/mesh/generators/CellBlockManager.hpp +++ b/src/coreComponents/mesh/generators/CellBlockManager.hpp @@ -23,7 +23,7 @@ #include "mesh/generators/FaceBlock.hpp" #include "mesh/generators/InternalWellGenerator.hpp" #include "mesh/generators/LineBlock.hpp" -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" #include "mesh/generators/CellBlockManagerABC.hpp" #include "mesh/generators/PartitionDescriptor.hpp" diff --git a/src/coreComponents/mesh/generators/CellBlockManagerABC.hpp b/src/coreComponents/mesh/generators/CellBlockManagerABC.hpp index d32f994a52e..3f46847704a 100644 --- a/src/coreComponents/mesh/generators/CellBlockManagerABC.hpp +++ b/src/coreComponents/mesh/generators/CellBlockManagerABC.hpp @@ -17,7 +17,7 @@ #include "CellBlockUtilities.hpp" #include "dataRepository/Group.hpp" -#include "LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" #include diff --git a/src/coreComponents/mesh/generators/LineBlock.hpp b/src/coreComponents/mesh/generators/LineBlock.hpp index d81463dd4a3..5a93c637fbd 100644 --- a/src/coreComponents/mesh/generators/LineBlock.hpp +++ b/src/coreComponents/mesh/generators/LineBlock.hpp @@ -15,7 +15,7 @@ #ifndef GEOSX_WELLBLOCK_HPP #define GEOSX_WELLBLOCK_HPP -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" namespace geos diff --git a/src/coreComponents/mesh/generators/MeshBlockMappings.cpp b/src/coreComponents/mesh/generators/MeshBlockMappings.cpp deleted file mode 100644 index 191071fc22e..00000000000 --- a/src/coreComponents/mesh/generators/MeshBlockMappings.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * ------------------------------------------------------------------------------------------------------------ - * SPDX-License-Identifier: LGPL-2.1-only - * - * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC - * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University - * Copyright (c) 2018-2020 TotalEnergies - * Copyright (c) 2020- GEOSX Contributors - * All rights reserved - * - * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. - * ------------------------------------------------------------------------------------------------------------ - */ - -#include "MeshBlockMappings.hpp" - -namespace geos -{ - -using namespace dataRepository; - -MeshBlockMappings::MeshBlockMappings( string const & name, - Group * const parent ) - : - CellBlockManagerABC( name, parent ) -{ - this->registerGroup< Group >( viewKeyStruct::cellBlocks() ); - this->registerGroup< Group >( viewKeyStruct::faceBlocks() ); - this->registerGroup< Group >( viewKeyStruct::lineBlocks() ); -} - -Group & MeshBlockMappings::getCellBlocks() -{ - return this->getGroup( viewKeyStruct::cellBlocks() ); -} - -Group & MeshBlockMappings::getFaceBlocks() -{ - return this->getGroup( viewKeyStruct::faceBlocks() ); -} - -LineBlockABC const & MeshBlockMappings::getLineBlock( string name ) const -{ - return this->getGroup( viewKeyStruct::lineBlocks() ).getGroup< LineBlockABC >( name ); -} - -Group const & MeshBlockMappings::getCellBlocks() const -{ - return this->getGroup( viewKeyStruct::cellBlocks() ); -} - -Group const & MeshBlockMappings::getFaceBlocks() const -{ - return this->getGroup( viewKeyStruct::faceBlocks() ); -} - -localIndex MeshBlockMappings::numNodes() const -{ - return m_numNodes; -} - -localIndex MeshBlockMappings::numEdges() const -{ - return m_numEdges; -} - -localIndex MeshBlockMappings::numFaces() const -{ - return m_numFaces; -} - -array2d< real64, nodes::REFERENCE_POSITION_PERM > MeshBlockMappings::getNodePositions() const -{ - return {}; -} - -ArrayOfArrays< localIndex > MeshBlockMappings::getNodeToEdges() const -{ - return {}; -} - -ArrayOfArrays< localIndex > MeshBlockMappings::getNodeToFaces() const -{ - return {}; -} - -ToCellRelation< ArrayOfArrays< localIndex>> MeshBlockMappings::getNodeToElements() const -{ - return {}; -} - -array2d< localIndex > MeshBlockMappings::getEdgeToNodes() const -{ - return {}; -} - -ArrayOfArrays< localIndex > MeshBlockMappings::getEdgeToFaces() const -{ - return {}; -} - -ArrayOfArrays< localIndex > MeshBlockMappings::getFaceToNodes() const -{ - return {}; -} - -ArrayOfArrays< localIndex > MeshBlockMappings::getFaceToEdges() const -{ - return {}; -} - -ToCellRelation< array2d< localIndex>> MeshBlockMappings::getFaceToElements() const -{ - return {}; -} - -array1d< globalIndex > MeshBlockMappings::getNodeLocalToGlobal() const -{ - return {}; -} - -std::map< string, SortedArray< localIndex>> const & MeshBlockMappings::getNodeSets() const -{ - return m_nodeSets; -} - -real64 MeshBlockMappings::getGlobalLength() const -{ - return m_globalLength; -} - -void MeshBlockMappings::generateHighOrderMaps( localIndex const order, - globalIndex const maxVertexGlobalID, - globalIndex const maxEdgeGlobalID, - globalIndex const maxFaceGlobalID, - arrayView1d< globalIndex const > const edgeLocalToGlobal, - arrayView1d< globalIndex const > const faceLocalToGlobal ) -{ - GEOS_ERROR( "`MeshBlockMappings::generateHighOrderMaps` is not implemented and should not be called." ); -} - -} // geos \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/MeshBlockMappings.hpp b/src/coreComponents/mesh/generators/MeshBlockMappings.hpp deleted file mode 100644 index 00fc352c020..00000000000 --- a/src/coreComponents/mesh/generators/MeshBlockMappings.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * ------------------------------------------------------------------------------------------------------------ - * SPDX-License-Identifier: LGPL-2.1-only - * - * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC - * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University - * Copyright (c) 2018-2020 TotalEnergies - * Copyright (c) 2020- GEOSX Contributors - * All rights reserved - * - * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. - * ------------------------------------------------------------------------------------------------------------ - */ - -#ifndef GEOS_GENERATORS_MESHBLOCKMAPPINGS_HPP -#define GEOS_GENERATORS_MESHBLOCKMAPPINGS_HPP - -#include "CellBlockManagerABC.hpp" - -namespace geos -{ - -class MeshBlockMappings: public CellBlockManagerABC -{ -public: - MeshBlockMappings( string const & name, Group * const parent ); - - virtual Group & getCellBlocks() override; - - virtual Group & getFaceBlocks() override; - - virtual LineBlockABC const & getLineBlock( string name ) const override; - - virtual Group const & getCellBlocks() const override; - - virtual Group const & getFaceBlocks() const override; - - virtual localIndex numNodes() const override; - - virtual localIndex numEdges() const override; - - virtual localIndex numFaces() const override; - - virtual array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override; - - virtual ArrayOfArrays< localIndex > getNodeToEdges() const override; - - virtual ArrayOfArrays< localIndex > getNodeToFaces() const override; - - virtual ToCellRelation< ArrayOfArrays< localIndex>> getNodeToElements() const override; - - virtual array2d< localIndex > getEdgeToNodes() const override; - - virtual ArrayOfArrays< localIndex > getEdgeToFaces() const override; - - virtual ArrayOfArrays< localIndex > getFaceToNodes() const override; - - virtual ArrayOfArrays< localIndex > getFaceToEdges() const override; - - virtual ToCellRelation< array2d< localIndex>> getFaceToElements() const override; - - virtual array1d< globalIndex > getNodeLocalToGlobal() const override; - - virtual std::map< string, SortedArray< localIndex>> const & getNodeSets() const override; - - virtual real64 getGlobalLength() const override; - - virtual void generateHighOrderMaps( localIndex const order, - globalIndex const maxVertexGlobalID, - globalIndex const maxEdgeGlobalID, - globalIndex const maxFaceGlobalID, - arrayView1d< globalIndex const > const edgeLocalToGlobal, - arrayView1d< globalIndex const > const faceLocalToGlobal ) override; - -private: - - struct viewKeyStruct - { - /// Cell blocks key - static constexpr char const * cellBlocks() - { return "cellBlocks"; } - - /// Face blocks key - static constexpr char const * faceBlocks() - { return "faceBlocks"; } - - /// Line blocks key - static constexpr char const * lineBlocks() - { return "lineBlocks"; } - }; - - std::map< string, SortedArray< localIndex > > m_nodeSets; - - real64 m_globalLength; - - localIndex m_numNodes; - localIndex m_numFaces; - localIndex m_numEdges; -}; - -} // geos - -#endif //GEOS_GENERATORS_MESHBLOCKMAPPINGS_HPP diff --git a/src/coreComponents/mesh/generators/WellGeneratorBase.cpp b/src/coreComponents/mesh/generators/WellGeneratorBase.cpp index 554f4530bf7..c38dba80f2e 100644 --- a/src/coreComponents/mesh/generators/WellGeneratorBase.cpp +++ b/src/coreComponents/mesh/generators/WellGeneratorBase.cpp @@ -15,7 +15,7 @@ #include "WellGeneratorBase.hpp" #include "mesh/mpiCommunications/CommunicationTools.hpp" #include "mesh/Perforation.hpp" -#include "mesh/generators/LineBlockABC.hpp" +#include "mesh/generators/include/LineBlockABC.hpp" #include "LvArray/src/genericTensorOps.hpp" namespace geos diff --git a/src/coreComponents/mesh/generators/include/CellMgr.hpp b/src/coreComponents/mesh/generators/include/CellMgr.hpp new file mode 100644 index 00000000000..38e188e4fad --- /dev/null +++ b/src/coreComponents/mesh/generators/include/CellMgr.hpp @@ -0,0 +1,69 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_CELLMGR_HPP +#define GEOS_CELLMGR_HPP + +#include "LineBlockABC.hpp" + +#include "dataRepository/Group.hpp" + +#include "common/DataTypes.hpp" + + +namespace geos::generators +{ + +class CellMgr +{ +public: + /** + * @brief Returns a group containing the cell blocks as @p CellBlockABC instances. + * @return Mutable reference to the cell blocks group. + * + * @note It should probably be better not to expose a non-const accessor here. + */ + virtual dataRepository::Group & getCellBlocks() = 0; + + /** + * @brief Returns a group containing the face blocks as @p FaceBlockABC instances. + * @return Mutable reference to the face blocks group. + * + * @note It should probably be better not to expose a non-const accessor here. + */ + virtual dataRepository::Group & getFaceBlocks() = 0; + + /** + * @brief Returns LineBlockABC corresponding to the given identifier + * @param name the name of the required LineBlockABC + * @return The LineBlockABC associated with the given name + */ + virtual LineBlockABC const & getLineBlock( string name ) const = 0; + + /** + * @brief Returns a group containing the cell blocks as CellBlockABC instances + * @return Const reference to the Group instance. + */ + virtual const dataRepository::Group & getCellBlocks() const = 0; + + /** + * @brief Returns a group containing the face blocks as FaceBlockABC instances + * @return Const reference to the Group instance. + */ + virtual const dataRepository::Group & getFaceBlocks() const = 0; +}; + +} + +#endif //GEOS_CELLMGR_HPP diff --git a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp new file mode 100644 index 00000000000..2dace04d03e --- /dev/null +++ b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp @@ -0,0 +1,49 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_EDGEMGR_HPP +#define GEOS_EDGEMGR_HPP + +#include "common/DataTypes.hpp" + +namespace geos::generators +{ + +class EdgeMgr +{ +public: + /** + * @brief Total number of edges across all the cell blocks. + * @return The total number of edges. + */ + virtual localIndex numEdges() const = 0; + + /** + * @brief Returns the edge to nodes mapping. + * @return A 1 to 2 relationship. The result is meant to have size (numEdges, 2). + */ + virtual array2d< localIndex > getEdgeToNodes() const = 0; + + /** + * @brief Returns the edge to faces mapping. + * @return A one to many relationship. + */ + virtual ArrayOfArrays< localIndex > getEdgeToFaces() const = 0; + + // TODO add the local -> global and the ghost rank. Use inheritance? +}; + +} + +#endif //GEOS_EDGEMGR_HPP diff --git a/src/coreComponents/mesh/generators/include/FaceMgr.hpp b/src/coreComponents/mesh/generators/include/FaceMgr.hpp new file mode 100644 index 00000000000..5e873669d63 --- /dev/null +++ b/src/coreComponents/mesh/generators/include/FaceMgr.hpp @@ -0,0 +1,57 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_FACEMGR_HPP +#define GEOS_FACEMGR_HPP + +#include "../CellBlockUtilities.hpp" // TODO At least part of it should become public. + +#include "common/DataTypes.hpp" + +namespace geos::generators +{ + +class FaceMgr +{ +public: + /** + * @brief Total number of faces across all the cell blocks. + * @return The total number of faces. + */ + virtual localIndex numFaces() const = 0; + + /** + * @brief Returns the face to nodes mapping. + * @return The one to many relationship. + */ + virtual ArrayOfArrays< localIndex > getFaceToNodes() const = 0; + + /** + * @brief Returns the face to edges mapping. + * @return A one to many relationship. + */ + virtual ArrayOfArrays< localIndex > getFaceToEdges() const = 0; + + /** + * @brief Returns the face to elements mapping. + * @return A 1 to 2 relationship. The result is meant to have size (numFaces, 2). + * + * In case the face only belongs to one single element, the second value of the table is -1. + */ + virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const = 0; +}; + +} + +#endif //GEOS_FACEMGR_HPP diff --git a/src/coreComponents/mesh/generators/LineBlockABC.hpp b/src/coreComponents/mesh/generators/include/LineBlockABC.hpp similarity index 98% rename from src/coreComponents/mesh/generators/LineBlockABC.hpp rename to src/coreComponents/mesh/generators/include/LineBlockABC.hpp index 723aaa69f7e..65a41de97be 100644 --- a/src/coreComponents/mesh/generators/LineBlockABC.hpp +++ b/src/coreComponents/mesh/generators/include/LineBlockABC.hpp @@ -20,7 +20,7 @@ #include -namespace geos +namespace geos // TODO Eventually add the `generators` { /** diff --git a/src/coreComponents/mesh/generators/include/MeshMappings.hpp b/src/coreComponents/mesh/generators/include/MeshMappings.hpp new file mode 100644 index 00000000000..6da8d0ebbbd --- /dev/null +++ b/src/coreComponents/mesh/generators/include/MeshMappings.hpp @@ -0,0 +1,92 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_MESHMAPPINGS_HPP +#define GEOS_MESHMAPPINGS_HPP + +#include "CellMgr.hpp" +#include "EdgeMgr.hpp" +#include "FaceMgr.hpp" +#include "NodeMgr.hpp" + +#include "dataRepository/Group.hpp" + +namespace geos::generators +{ + +class MeshMappings : public dataRepository::Group +{ +public: + /** + * @brief Constructor + * @param name The name of this Group. + * @param parent The parent Group. + */ + MeshMappings( string const & name, Group * const parent ): + Group( name, parent ) + { + // Left blank + } + + /** + * @brief Extra space for node to edges mapping. + * @return Number of extra values as an integer. + */ + static constexpr localIndex edgeMapExtraSpacePerNode() + { return 8; } + + /** + * @brief Extra space for node to faces mapping. + * @return Number of extra values as an integer. + */ + static constexpr localIndex faceMapExtraSpacePerNode() + { return 8; } + + /** + * @brief Extra space for node to elements mapping. + * @return Number of extra values as an integer. + */ + static constexpr localIndex elemMapExtraSpacePerNode() + { return 8; } + + /** + * @brief Extra space for extra nodes. + * @return Number of extra values as an integer. + */ + static constexpr localIndex nodeMapExtraSpacePerFace() + { return 4; } + + /** + * @brief Extra space for extra faces. + * @return Number of extra values as an integer. + */ + static constexpr localIndex edgeMapExtraSpacePerFace() + { return 4; } + + /** + * @brief Extra space for extra edges. + * @return Number of extra values as an integer. + */ + static constexpr localIndex faceMapExtraSpacePerEdge() + { return 4; } + + virtual CellMgr const & getCellMgr() const = 0 ; + virtual EdgeMgr const & getEdgeMgr() const = 0 ; + virtual FaceMgr const & getFaceMgr() const = 0 ; + virtual NodeMgr const & getNodeMgr() const = 0 ; +}; + +} + +#endif //GEOS_MESHMAPPINGS_HPP diff --git a/src/coreComponents/mesh/generators/include/NodeMgr.hpp b/src/coreComponents/mesh/generators/include/NodeMgr.hpp new file mode 100644 index 00000000000..6b814538eac --- /dev/null +++ b/src/coreComponents/mesh/generators/include/NodeMgr.hpp @@ -0,0 +1,76 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_NODEMGR_HPP +#define GEOS_NODEMGR_HPP + + +#include "../CellBlockUtilities.hpp" // TODO At least part of it should become public. + +#include "common/DataTypes.hpp" + +namespace geos::generators +{ + +class NodeMgr +{ +public: + /** + * @brief Total number of nodes across all the cell blocks. + * @return The total number of nodes. + * + * Nodes shared by multiple cell blocks are counted only once. + */ + virtual localIndex numNodes() const = 0; + + /** + * @brief Returns the node coordinates in a (numNodes, 3) 2d array. + * @return A const view to the array. + */ + virtual array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const = 0; + + /** + * @brief Returns the node to edges mapping. + * @return The one to many relationship. + */ + virtual ArrayOfArrays< localIndex > getNodeToEdges() const = 0; + + /** + * @brief Returns the face to nodes mappings. + * @return The one to many relationship. + */ + virtual ArrayOfArrays< localIndex > getNodeToFaces() const = 0; + + /** + * @brief Returns the node to elements mapping. + * @return A one to many relationship. + */ + virtual ToCellRelation< ArrayOfArrays< localIndex > > getNodeToElements() const = 0; + + /** + * @brief The node to global mapping for nodes. + * @return The mapping as an array of size numNodes. + */ + virtual array1d< globalIndex > getLocalToGlobal() const = 0; + + /** + * @brief Returns the node sets. Key of the map is the name of the set. + * @return A reference to constant map. + */ + virtual std::map< string, SortedArray< localIndex > > const & getNodeSets() const = 0; +}; + +} + +#endif //GEOS_NODEMGR_HPP From b6f30c99833e69573880bc3f8dca84377422b9b1 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 14 May 2024 17:10:04 -0700 Subject: [PATCH 050/106] Move CellBlockABC and FaceBlockABC to the include folder. --- src/coreComponents/mesh/CMakeLists.txt | 4 ++-- src/coreComponents/mesh/CellElementRegion.cpp | 2 +- src/coreComponents/mesh/CellElementSubRegion.hpp | 2 +- src/coreComponents/mesh/FaceElementSubRegion.hpp | 2 +- src/coreComponents/mesh/generators/CellBlock.hpp | 2 +- src/coreComponents/mesh/generators/FaceBlock.hpp | 2 +- .../mesh/generators/{ => include}/CellBlockABC.hpp | 2 +- .../mesh/generators/{ => include}/FaceBlockABC.hpp | 5 +++-- src/coreComponents/unitTests/meshTests/testVTKImport.cpp | 2 +- 9 files changed, 12 insertions(+), 11 deletions(-) rename src/coreComponents/mesh/generators/{ => include}/CellBlockABC.hpp (98%) rename src/coreComponents/mesh/generators/{ => include}/FaceBlockABC.hpp (98%) diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index 59bdb77afd9..34f63b42f25 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -42,7 +42,7 @@ set( mesh_headers WellElementRegion.hpp WellElementSubRegion.hpp generators/CellBlock.hpp - generators/CellBlockABC.hpp + generators/include/CellBlockABC.hpp generators/CellBlockManager.hpp generators/CellBlockManagerABC.hpp generators/CellBlockUtilities.hpp @@ -50,7 +50,7 @@ set( mesh_headers generators/include/LineBlockABC.hpp generators/ExternalMeshGeneratorBase.hpp generators/FaceBlock.hpp - generators/FaceBlockABC.hpp + generators/include/FaceBlockABC.hpp generators/InternalMeshGenerator.hpp generators/InternalWellGenerator.hpp generators/InternalWellboreGenerator.hpp diff --git a/src/coreComponents/mesh/CellElementRegion.cpp b/src/coreComponents/mesh/CellElementRegion.cpp index 3d614db6606..f10494109b0 100644 --- a/src/coreComponents/mesh/CellElementRegion.cpp +++ b/src/coreComponents/mesh/CellElementRegion.cpp @@ -14,7 +14,7 @@ #include "CellElementRegion.hpp" #include "CellElementSubRegion.hpp" -#include "mesh/generators/CellBlockABC.hpp" +#include "mesh/generators/include/CellBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/CellElementSubRegion.hpp b/src/coreComponents/mesh/CellElementSubRegion.hpp index 37e6c3537e2..758a8fa0913 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.hpp +++ b/src/coreComponents/mesh/CellElementSubRegion.hpp @@ -16,7 +16,7 @@ #ifndef GEOS_MESH_CELLELEMENTSUBREGION_HPP_ #define GEOS_MESH_CELLELEMENTSUBREGION_HPP_ -#include "mesh/generators/CellBlockABC.hpp" +#include "mesh/generators/include/CellBlockABC.hpp" #include "mesh/NodeManager.hpp" #include "mesh/FaceManager.hpp" #include "mesh/utilities/ComputationalGeometry.hpp" diff --git a/src/coreComponents/mesh/FaceElementSubRegion.hpp b/src/coreComponents/mesh/FaceElementSubRegion.hpp index 4823f96ce8b..963e485c994 100644 --- a/src/coreComponents/mesh/FaceElementSubRegion.hpp +++ b/src/coreComponents/mesh/FaceElementSubRegion.hpp @@ -20,7 +20,7 @@ #define GEOS_MESH_FACEELEMENTSUBREGION_HPP_ #include "SurfaceElementSubRegion.hpp" -#include "mesh/generators/FaceBlockABC.hpp" +#include "mesh/generators/include/FaceBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/generators/CellBlock.hpp b/src/coreComponents/mesh/generators/CellBlock.hpp index 01fa2124bd8..ecc270c0ab0 100644 --- a/src/coreComponents/mesh/generators/CellBlock.hpp +++ b/src/coreComponents/mesh/generators/CellBlock.hpp @@ -16,7 +16,7 @@ #define GEOS_MESH_CELLBLOCK_HPP_ #include "dataRepository/Group.hpp" -#include "mesh/generators/CellBlockABC.hpp" +#include "mesh/generators/include/CellBlockABC.hpp" #include "mesh/ElementType.hpp" namespace geos diff --git a/src/coreComponents/mesh/generators/FaceBlock.hpp b/src/coreComponents/mesh/generators/FaceBlock.hpp index 5788cf0d154..87c59e8902f 100644 --- a/src/coreComponents/mesh/generators/FaceBlock.hpp +++ b/src/coreComponents/mesh/generators/FaceBlock.hpp @@ -15,7 +15,7 @@ #ifndef GEOS_FACEBLOCK_HPP #define GEOS_FACEBLOCK_HPP -#include "FaceBlockABC.hpp" +#include "mesh/generators/include/FaceBlockABC.hpp" namespace geos { diff --git a/src/coreComponents/mesh/generators/CellBlockABC.hpp b/src/coreComponents/mesh/generators/include/CellBlockABC.hpp similarity index 98% rename from src/coreComponents/mesh/generators/CellBlockABC.hpp rename to src/coreComponents/mesh/generators/include/CellBlockABC.hpp index fadde416894..1d9ba20d388 100644 --- a/src/coreComponents/mesh/generators/CellBlockABC.hpp +++ b/src/coreComponents/mesh/generators/include/CellBlockABC.hpp @@ -21,7 +21,7 @@ #include -namespace geos +namespace geos // TODO eventually add the `generators` namespace { /** diff --git a/src/coreComponents/mesh/generators/FaceBlockABC.hpp b/src/coreComponents/mesh/generators/include/FaceBlockABC.hpp similarity index 98% rename from src/coreComponents/mesh/generators/FaceBlockABC.hpp rename to src/coreComponents/mesh/generators/include/FaceBlockABC.hpp index fb4bc201e07..d766dc1c941 100644 --- a/src/coreComponents/mesh/generators/FaceBlockABC.hpp +++ b/src/coreComponents/mesh/generators/include/FaceBlockABC.hpp @@ -15,11 +15,12 @@ #ifndef GEOS_FACEBLOCKABC_HPP #define GEOS_FACEBLOCKABC_HPP -#include "CellBlockUtilities.hpp" +#include "../CellBlockUtilities.hpp" // TODO make at least part of this public + #include "dataRepository/Group.hpp" #include "common/DataTypes.hpp" -namespace geos +namespace geos // TODO eventually add the `generators` namespace { /** diff --git a/src/coreComponents/unitTests/meshTests/testVTKImport.cpp b/src/coreComponents/unitTests/meshTests/testVTKImport.cpp index 39a333b7e84..cd3a9c00384 100644 --- a/src/coreComponents/unitTests/meshTests/testVTKImport.cpp +++ b/src/coreComponents/unitTests/meshTests/testVTKImport.cpp @@ -20,7 +20,7 @@ #include "mainInterface/initialization.hpp" #include "mesh/MeshManager.hpp" #include "mesh/generators/CellBlockManagerABC.hpp" -#include "mesh/generators/CellBlockABC.hpp" +#include "mesh/generators/include/CellBlockABC.hpp" #include "mesh/generators/VTKUtilities.hpp" // special CMake-generated include From 4bacf1974a3d631cc099ffd54c8bba0d69869be6 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 14 May 2024 18:18:28 -0700 Subject: [PATCH 051/106] Have doTheNewGhosting return a pointer to the mesh data. --- src/coreComponents/mesh/generators/NewGhosting.cpp | 10 ++++++---- src/coreComponents/mesh/generators/NewGhosting.hpp | 6 ++++-- .../mesh/generators/VTKMeshGenerator.cpp | 4 +++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index cb5b4f0615e..15b63de4a25 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1148,8 +1148,8 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, } -void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< MpiRank > const & neighbors ) +std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< MpiRank > const & neighbors ) { // Now we exchange the data with our neighbors. MpiRank const curRank{ MpiWrapper::commRank() }; @@ -1220,10 +1220,12 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, // } assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); + + return {}; } -void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< int > const & neighbors ) +std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors ) { std::set< MpiRank > neighbors_; for( int const & rank: neighbors ) diff --git a/src/coreComponents/mesh/generators/NewGhosting.hpp b/src/coreComponents/mesh/generators/NewGhosting.hpp index b908d9cfb2c..b077e514126 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.hpp +++ b/src/coreComponents/mesh/generators/NewGhosting.hpp @@ -15,6 +15,8 @@ #ifndef GEOS_NEWGHOSTING_HPP #define GEOS_NEWGHOSTING_HPP +#include "include/MeshMappings.hpp" + #include #include @@ -23,8 +25,8 @@ namespace geos::ghosting { -void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< int > const & neighbors ); +std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors ); } // end of namespace ghosting diff --git a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp index 72ebf48c0d1..3fe5fdd34dc 100644 --- a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp +++ b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp @@ -22,6 +22,8 @@ #include "mesh/generators/VTKMeshGeneratorTools.hpp" #include "mesh/generators/CellBlockManager.hpp" #include "mesh/generators/NewGhosting.hpp" + +#include "include/MeshMappings.hpp" #include "common/DataTypes.hpp" #include @@ -111,7 +113,7 @@ void VTKMeshGenerator::fillCellBlockManager( CellBlockManager & cellBlockManager if( m_useNewGhosting ) { GEOS_LOG_RANK( "Here we go, new ghosting!" ); - ghosting::doTheNewGhosting( m_vtkMesh, partition.getMetisNeighborList() ); + std::unique_ptr< generators::MeshMappings > mm = ghosting::doTheNewGhosting( m_vtkMesh, partition.getMetisNeighborList() ); } GEOS_LOG_LEVEL_RANK_0( 2, " preprocessing..." ); From d722a881ce32d4810248e92034351241f33d587d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 10:19:40 -0700 Subject: [PATCH 052/106] Move indices to their own file --- src/coreComponents/mesh/CMakeLists.txt | 11 +- .../mesh/generators/Indices.hpp | 121 ++++++++++++++++++ .../mesh/generators/NewGhosting.cpp | 96 +------------- src/coreComponents/mesh/generators/Pods.cpp | 19 +++ src/coreComponents/mesh/generators/Pods.hpp | 28 ++++ 5 files changed, 176 insertions(+), 99 deletions(-) create mode 100644 src/coreComponents/mesh/generators/Indices.hpp create mode 100644 src/coreComponents/mesh/generators/Pods.cpp create mode 100644 src/coreComponents/mesh/generators/Pods.hpp diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index 34f63b42f25..be5ea58c8d9 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -42,25 +42,27 @@ set( mesh_headers WellElementRegion.hpp WellElementSubRegion.hpp generators/CellBlock.hpp - generators/include/CellBlockABC.hpp generators/CellBlockManager.hpp generators/CellBlockManagerABC.hpp generators/CellBlockUtilities.hpp - generators/LineBlock.hpp - generators/include/LineBlockABC.hpp generators/ExternalMeshGeneratorBase.hpp generators/FaceBlock.hpp - generators/include/FaceBlockABC.hpp + generators/Indices.hpp generators/InternalMeshGenerator.hpp generators/InternalWellGenerator.hpp generators/InternalWellboreGenerator.hpp + generators/LineBlock.hpp generators/MeshGeneratorBase.hpp generators/ParMETISInterface.hpp generators/ParticleMeshGenerator.hpp generators/PartitionDescriptor.hpp + generators/Pods.hpp generators/PrismUtilities.hpp generators/WellGeneratorABC.hpp generators/WellGeneratorBase.hpp + generators/include/CellBlockABC.hpp + generators/include/FaceBlockABC.hpp + generators/include/LineBlockABC.hpp mpiCommunications/CommID.hpp mpiCommunications/CommunicationTools.hpp mpiCommunications/MPI_iCommData.hpp @@ -130,6 +132,7 @@ set( mesh_sources generators/MeshGeneratorBase.cpp generators/ParMETISInterface.cpp generators/ParticleMeshGenerator.cpp + generators/Pods.cpp generators/WellGeneratorBase.cpp mpiCommunications/CommID.cpp mpiCommunications/CommunicationTools.cpp diff --git a/src/coreComponents/mesh/generators/Indices.hpp b/src/coreComponents/mesh/generators/Indices.hpp new file mode 100644 index 00000000000..799626e2b3d --- /dev/null +++ b/src/coreComponents/mesh/generators/Indices.hpp @@ -0,0 +1,121 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_INDICES_HPP +#define GEOS_INDICES_HPP + +#include "common/GeosxMacros.hpp" + +#include "LvArray/src/limits.hpp" + +// TODO remove in the end. +#include +using json = nlohmann::json; + +// TODO make conditional in the end. +#include + +namespace geos +{ + +template< typename OUTPUT, typename INPUT > +inline GEOS_HOST_DEVICE +OUTPUT intConv( INPUT input ) +{ + return LvArray::integerConversion< OUTPUT >( input ); +} + +using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; +using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; +using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; +using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; +using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; +using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; + +using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; + +EdgeGlbIdx operator "" _egi( unsigned long long int i ) +{ + return EdgeGlbIdx{ EdgeGlbIdx::UnderlyingType( i ) }; +} + +FaceGlbIdx operator "" _fgi( unsigned long long int i ) +{ + return FaceGlbIdx{ FaceGlbIdx::UnderlyingType( i ) }; +} + +MpiRank operator "" _mpi( unsigned long long int i ) +{ + return MpiRank{ MpiRank::UnderlyingType( i ) }; +} + +void to_json( json & j, + const MpiRank & v ) +{ + j = v.get(); +} + +void from_json( const json & j, + MpiRank & v ) +{ + v = MpiRank{ j.get< MpiRank::UnderlyingType >() }; // TODO use a `traits` instead +} + +void from_json( const json & j, + NodeGlbIdx & v ) +{ + v = NodeGlbIdx{ j.get< NodeGlbIdx::UnderlyingType >() }; +} + +void to_json( json & j, + const NodeGlbIdx & v ) +{ + j = v.get(); +} + +void to_json( json & j, + const EdgeGlbIdx & v ) +{ + j = v.get(); +} + +void from_json( const json & j, + EdgeGlbIdx & v ) +{ + v = EdgeGlbIdx{ j.get< EdgeGlbIdx::UnderlyingType >() }; +} + +void to_json( json & j, + const FaceGlbIdx & v ) +{ + j = v.get(); +} + +void from_json( const json & j, + FaceGlbIdx & v ) +{ + v = FaceGlbIdx{ j.get< FaceGlbIdx::UnderlyingType >() }; +} + +void to_json( json & j, + const CellGlbIdx & v ) +{ + j = v.get(); +} + +} + +#endif //GEOS_INDICES_HPP diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 15b63de4a25..714636a52e2 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -28,8 +28,7 @@ #include #include - -#include +#include "Indices.hpp" #include #include @@ -43,102 +42,9 @@ using json = nlohmann::json; #include #include -namespace geos -{ - -template< typename OUTPUT, typename INPUT > -inline GEOS_HOST_DEVICE -OUTPUT intConv( INPUT input ) -{ - return LvArray::integerConversion< OUTPUT >( input ); -} - -} // end of namespace - - namespace geos::ghosting { -using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; -using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; -using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; -using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; -using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; -using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; - -using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; - -EdgeGlbIdx operator "" _egi( unsigned long long int i ) -{ - return EdgeGlbIdx{ EdgeGlbIdx::UnderlyingType( i ) }; -} - -FaceGlbIdx operator "" _fgi( unsigned long long int i ) -{ - return FaceGlbIdx{ FaceGlbIdx::UnderlyingType( i ) }; -} - -MpiRank operator "" _mpi( unsigned long long int i ) -{ - return MpiRank{ MpiRank::UnderlyingType( i ) }; -} - -void to_json( json & j, - const MpiRank & v ) -{ - j = v.get(); -} - -void from_json( const json & j, - MpiRank & v ) -{ - v = MpiRank{ j.get< MpiRank::UnderlyingType >() }; // TODO use a `traits` instead -} - -void from_json( const json & j, - NodeGlbIdx & v ) -{ - v = NodeGlbIdx{ j.get< NodeGlbIdx::UnderlyingType >() }; -} - -void to_json( json & j, - const NodeGlbIdx & v ) -{ - j = v.get(); -} - -void to_json( json & j, - const EdgeGlbIdx & v ) -{ - j = v.get(); -} - -void from_json( const json & j, - EdgeGlbIdx & v ) -{ - v = EdgeGlbIdx{ j.get< EdgeGlbIdx::UnderlyingType >() }; -} - -void to_json( json & j, - const FaceGlbIdx & v ) -{ - j = v.get(); -} - -void from_json( const json & j, - FaceGlbIdx & v ) -{ - v = FaceGlbIdx{ j.get< FaceGlbIdx::UnderlyingType >() }; -} - -void to_json( json & j, - const CellGlbIdx & v ) -{ - j = v.get(); -} - using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; using Face = std::vector< NodeGlbIdx >; diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp new file mode 100644 index 00000000000..bae191c4cda --- /dev/null +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -0,0 +1,19 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 Total, S.A + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "Pods.hpp" + +namespace geos +{ +} // geos \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp new file mode 100644 index 00000000000..a598f8371df --- /dev/null +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -0,0 +1,28 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 Total, S.A + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_PODS_HPP +#define GEOS_PODS_HPP + +namespace geos +{ + +class Pods +{ + +}; + +} // geos + +#endif //GEOS_PODS_HPP From 7cf2c53001caa4d894f37a0d42ebd7363336c481 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 10:58:14 -0700 Subject: [PATCH 053/106] Fist incomplete impl of the new public interface of the module. --- .../mesh/generators/Indices.hpp | 73 +++++++++++----- .../mesh/generators/NewGhosting.cpp | 6 ++ src/coreComponents/mesh/generators/Pods.cpp | 83 +++++++++++++++++++ src/coreComponents/mesh/generators/Pods.hpp | 60 +++++++++++++- 4 files changed, 200 insertions(+), 22 deletions(-) diff --git a/src/coreComponents/mesh/generators/Indices.hpp b/src/coreComponents/mesh/generators/Indices.hpp index 799626e2b3d..9c1e9765993 100644 --- a/src/coreComponents/mesh/generators/Indices.hpp +++ b/src/coreComponents/mesh/generators/Indices.hpp @@ -16,6 +16,7 @@ #define GEOS_INDICES_HPP #include "common/GeosxMacros.hpp" +#include "common/DataTypes.hpp" #include "LvArray/src/limits.hpp" @@ -47,71 +48,101 @@ using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent: using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; -EdgeGlbIdx operator "" _egi( unsigned long long int i ) +inline NodeLocIdx operator "" _nli( unsigned long long int i ) +{ + return NodeLocIdx{ NodeLocIdx::UnderlyingType( i ) }; +} + +inline EdgeLocIdx operator "" _eli( unsigned long long int i ) +{ + return EdgeLocIdx{ EdgeLocIdx::UnderlyingType( i ) }; +} + +inline FaceLocIdx operator "" _fli( unsigned long long int i ) +{ + return FaceLocIdx{ FaceLocIdx::UnderlyingType( i ) }; +} + +inline CellLocIdx operator "" _cli( unsigned long long int i ) +{ + return CellLocIdx{ CellLocIdx::UnderlyingType( i ) }; +} + +inline NodeGlbIdx operator "" _ngi( unsigned long long int i ) +{ + return NodeGlbIdx{ NodeGlbIdx::UnderlyingType( i ) }; +} + +inline EdgeGlbIdx operator "" _egi( unsigned long long int i ) { return EdgeGlbIdx{ EdgeGlbIdx::UnderlyingType( i ) }; } -FaceGlbIdx operator "" _fgi( unsigned long long int i ) +inline FaceGlbIdx operator "" _fgi( unsigned long long int i ) { return FaceGlbIdx{ FaceGlbIdx::UnderlyingType( i ) }; } -MpiRank operator "" _mpi( unsigned long long int i ) +inline CellGlbIdx operator "" _cgi( unsigned long long int i ) +{ + return CellGlbIdx{ CellGlbIdx::UnderlyingType( i ) }; +} + +inline MpiRank operator "" _mpi( unsigned long long int i ) { return MpiRank{ MpiRank::UnderlyingType( i ) }; } -void to_json( json & j, - const MpiRank & v ) +inline void to_json( json & j, + const MpiRank & v ) { j = v.get(); } -void from_json( const json & j, - MpiRank & v ) +inline void from_json( const json & j, + MpiRank & v ) { v = MpiRank{ j.get< MpiRank::UnderlyingType >() }; // TODO use a `traits` instead } -void from_json( const json & j, - NodeGlbIdx & v ) +inline void from_json( const json & j, + NodeGlbIdx & v ) { v = NodeGlbIdx{ j.get< NodeGlbIdx::UnderlyingType >() }; } -void to_json( json & j, - const NodeGlbIdx & v ) +inline void to_json( json & j, + const NodeGlbIdx & v ) { j = v.get(); } -void to_json( json & j, - const EdgeGlbIdx & v ) +inline void to_json( json & j, + const EdgeGlbIdx & v ) { j = v.get(); } -void from_json( const json & j, - EdgeGlbIdx & v ) +inline void from_json( const json & j, + EdgeGlbIdx & v ) { v = EdgeGlbIdx{ j.get< EdgeGlbIdx::UnderlyingType >() }; } -void to_json( json & j, - const FaceGlbIdx & v ) +inline void to_json( json & j, + const FaceGlbIdx & v ) { j = v.get(); } -void from_json( const json & j, - FaceGlbIdx & v ) +inline void from_json( const json & j, + FaceGlbIdx & v ) { v = FaceGlbIdx{ j.get< FaceGlbIdx::UnderlyingType >() }; } -void to_json( json & j, - const CellGlbIdx & v ) +inline void to_json( json & j, + const CellGlbIdx & v ) { j = v.get(); } diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 714636a52e2..c356c7f485b 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -18,6 +18,8 @@ #include "mesh/mpiCommunications/MPI_iCommData.hpp" #include "mesh/mpiCommunications/NeighborCommunicator.hpp" +#include "Pods.hpp" + #include "common/MpiWrapper.hpp" #include "common/DataTypes.hpp" @@ -1127,6 +1129,10 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); + NodeMgrImpl const nodeMgr( 0_nli ); + EdgeMgrImpl const edgeMgr( 0_eli ); + FaceMgrImpl const faceMgr( 0_fli ); + return {}; } diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index bae191c4cda..10307cfcfee 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -16,4 +16,87 @@ namespace geos { + +NodeMgrImpl::NodeMgrImpl( NodeLocIdx const & numNodes ) + : m_numNodes( numNodes ) +{ } + +localIndex NodeMgrImpl::numNodes() const +{ + return intConv< localIndex >( m_numNodes.get() ); +} + +array2d< real64, nodes::REFERENCE_POSITION_PERM > NodeMgrImpl::getNodePositions() const +{ + return {}; +} + +ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToEdges() const +{ + return {}; +} + +ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToFaces() const +{ + return {}; +} + +ToCellRelation< ArrayOfArrays< localIndex > > NodeMgrImpl::getNodeToElements() const +{ + return {}; +} + +array1d< globalIndex > NodeMgrImpl::getLocalToGlobal() const +{ + return {}; +} + +std::map< string, SortedArray< localIndex > > const & NodeMgrImpl::getNodeSets() const +{ + return m_todo; +} + +EdgeMgrImpl::EdgeMgrImpl( EdgeLocIdx const & numEdges ) + : m_numEdges( numEdges ) +{ } + +localIndex EdgeMgrImpl::numEdges() const +{ + return intConv< localIndex >( m_numEdges.get() ); +} + +array2d< localIndex > EdgeMgrImpl::getEdgeToNodes() const +{ + return {}; +} + +ArrayOfArrays< localIndex > EdgeMgrImpl::getEdgeToFaces() const +{ + return {}; +} + +FaceMgrImpl::FaceMgrImpl( FaceLocIdx const & numFaces ) + : m_numFaces( numFaces ) +{ } + +localIndex FaceMgrImpl::numFaces() const +{ + return intConv< localIndex >( m_numFaces.get() ); +} + +ArrayOfArrays< localIndex > FaceMgrImpl::getFaceToNodes() const +{ + return {}; +} + +ArrayOfArrays< localIndex > FaceMgrImpl::getFaceToEdges() const +{ + return {}; +} + +ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const +{ + return {}; +} + } // geos \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index a598f8371df..5c8f2676912 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -15,12 +15,70 @@ #ifndef GEOS_PODS_HPP #define GEOS_PODS_HPP +#include "Indices.hpp" + +#include "include/NodeMgr.hpp" +#include "include/EdgeMgr.hpp" +#include "include/FaceMgr.hpp" +#include "include/CellMgr.hpp" + namespace geos { -class Pods +class NodeMgrImpl : public generators::NodeMgr +{ +public: + explicit NodeMgrImpl( NodeLocIdx const & numNodes ); + + localIndex numNodes() const override; + + array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override; + + ArrayOfArrays< localIndex > getNodeToEdges() const override; + + ArrayOfArrays< localIndex > getNodeToFaces() const override; + + ToCellRelation< ArrayOfArrays< localIndex>> getNodeToElements() const override; + + array1d< globalIndex > getLocalToGlobal() const override; + + std::map< string, SortedArray< localIndex > > const & getNodeSets() const override; + +private: + NodeLocIdx m_numNodes; + std::map< string, SortedArray< localIndex > > m_todo; +}; + +class EdgeMgrImpl : public generators::EdgeMgr +{ +public: + explicit EdgeMgrImpl( EdgeLocIdx const & numEdges ); + + virtual localIndex numEdges() const override; + + virtual array2d< localIndex > getEdgeToNodes() const override; + + virtual ArrayOfArrays< localIndex > getEdgeToFaces() const override; + +private: + EdgeLocIdx m_numEdges; +}; + +class FaceMgrImpl : public generators::FaceMgr { +public: + explicit FaceMgrImpl( FaceLocIdx const & numFaces ); + + virtual localIndex numFaces() const override; + + virtual ArrayOfArrays< localIndex > getFaceToNodes() const override; + + virtual ArrayOfArrays< localIndex > getFaceToEdges() const override; + + virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const override; +private: + FaceLocIdx m_numFaces; }; } // geos From c5ccbf6ceb43440ad3d066d2fccf4738349186aa Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 12:00:28 -0700 Subject: [PATCH 054/106] Add nodiscard and ghost rank for the EdgeMgr. --- src/coreComponents/mesh/generators/Pods.cpp | 5 +++++ src/coreComponents/mesh/generators/Pods.hpp | 16 +++++++++------- .../mesh/generators/include/EdgeMgr.hpp | 12 +++++++++--- .../mesh/generators/include/FaceMgr.hpp | 8 ++++---- .../mesh/generators/include/NodeMgr.hpp | 14 +++++++------- 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 10307cfcfee..24ee9f0d274 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -75,6 +75,11 @@ ArrayOfArrays< localIndex > EdgeMgrImpl::getEdgeToFaces() const return {}; } +array1d< integer > EdgeMgrImpl::getGhostRank() const +{ + return {}; +} + FaceMgrImpl::FaceMgrImpl( FaceLocIdx const & numFaces ) : m_numFaces( numFaces ) { } diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 5c8f2676912..2a2f89573f2 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -54,11 +54,13 @@ class EdgeMgrImpl : public generators::EdgeMgr public: explicit EdgeMgrImpl( EdgeLocIdx const & numEdges ); - virtual localIndex numEdges() const override; + localIndex numEdges() const override; - virtual array2d< localIndex > getEdgeToNodes() const override; + array2d< localIndex > getEdgeToNodes() const override; - virtual ArrayOfArrays< localIndex > getEdgeToFaces() const override; + ArrayOfArrays< localIndex > getEdgeToFaces() const override; + + array1d< integer > getGhostRank() const override; private: EdgeLocIdx m_numEdges; @@ -69,13 +71,13 @@ class FaceMgrImpl : public generators::FaceMgr public: explicit FaceMgrImpl( FaceLocIdx const & numFaces ); - virtual localIndex numFaces() const override; + localIndex numFaces() const override; - virtual ArrayOfArrays< localIndex > getFaceToNodes() const override; + ArrayOfArrays< localIndex > getFaceToNodes() const override; - virtual ArrayOfArrays< localIndex > getFaceToEdges() const override; + ArrayOfArrays< localIndex > getFaceToEdges() const override; - virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const override; + ToCellRelation< array2d< localIndex > > getFaceToElements() const override; private: FaceLocIdx m_numFaces; diff --git a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp index 2dace04d03e..654525194e9 100644 --- a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp +++ b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp @@ -27,19 +27,25 @@ class EdgeMgr * @brief Total number of edges across all the cell blocks. * @return The total number of edges. */ - virtual localIndex numEdges() const = 0; + [[nodiscard]] virtual localIndex numEdges() const = 0; /** * @brief Returns the edge to nodes mapping. * @return A 1 to 2 relationship. The result is meant to have size (numEdges, 2). */ - virtual array2d< localIndex > getEdgeToNodes() const = 0; + [[nodiscard]] virtual array2d< localIndex > getEdgeToNodes() const = 0; /** * @brief Returns the edge to faces mapping. * @return A one to many relationship. */ - virtual ArrayOfArrays< localIndex > getEdgeToFaces() const = 0; + [[nodiscard]] virtual ArrayOfArrays< localIndex > getEdgeToFaces() const = 0; + + /** + * @brief Returns the ghost rank mapping. Index is an edge index local to the MPI rank. + * @return A @c numEdges length array. + */ + [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; // TODO add the local -> global and the ghost rank. Use inheritance? }; diff --git a/src/coreComponents/mesh/generators/include/FaceMgr.hpp b/src/coreComponents/mesh/generators/include/FaceMgr.hpp index 5e873669d63..439c1e7283f 100644 --- a/src/coreComponents/mesh/generators/include/FaceMgr.hpp +++ b/src/coreComponents/mesh/generators/include/FaceMgr.hpp @@ -29,19 +29,19 @@ class FaceMgr * @brief Total number of faces across all the cell blocks. * @return The total number of faces. */ - virtual localIndex numFaces() const = 0; + [[nodiscard]] virtual localIndex numFaces() const = 0; /** * @brief Returns the face to nodes mapping. * @return The one to many relationship. */ - virtual ArrayOfArrays< localIndex > getFaceToNodes() const = 0; + [[nodiscard]] virtual ArrayOfArrays< localIndex > getFaceToNodes() const = 0; /** * @brief Returns the face to edges mapping. * @return A one to many relationship. */ - virtual ArrayOfArrays< localIndex > getFaceToEdges() const = 0; + [[nodiscard]] virtual ArrayOfArrays< localIndex > getFaceToEdges() const = 0; /** * @brief Returns the face to elements mapping. @@ -49,7 +49,7 @@ class FaceMgr * * In case the face only belongs to one single element, the second value of the table is -1. */ - virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const = 0; + [[nodiscard]] virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const = 0; }; } diff --git a/src/coreComponents/mesh/generators/include/NodeMgr.hpp b/src/coreComponents/mesh/generators/include/NodeMgr.hpp index 6b814538eac..0ba69c1383b 100644 --- a/src/coreComponents/mesh/generators/include/NodeMgr.hpp +++ b/src/coreComponents/mesh/generators/include/NodeMgr.hpp @@ -32,43 +32,43 @@ class NodeMgr * * Nodes shared by multiple cell blocks are counted only once. */ - virtual localIndex numNodes() const = 0; + [[nodiscard]] virtual localIndex numNodes() const = 0; /** * @brief Returns the node coordinates in a (numNodes, 3) 2d array. * @return A const view to the array. */ - virtual array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const = 0; + [[nodiscard]] virtual array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const = 0; /** * @brief Returns the node to edges mapping. * @return The one to many relationship. */ - virtual ArrayOfArrays< localIndex > getNodeToEdges() const = 0; + [[nodiscard]] virtual ArrayOfArrays< localIndex > getNodeToEdges() const = 0; /** * @brief Returns the face to nodes mappings. * @return The one to many relationship. */ - virtual ArrayOfArrays< localIndex > getNodeToFaces() const = 0; + [[nodiscard]] virtual ArrayOfArrays< localIndex > getNodeToFaces() const = 0; /** * @brief Returns the node to elements mapping. * @return A one to many relationship. */ - virtual ToCellRelation< ArrayOfArrays< localIndex > > getNodeToElements() const = 0; + [[nodiscard]] virtual ToCellRelation< ArrayOfArrays< localIndex > > getNodeToElements() const = 0; /** * @brief The node to global mapping for nodes. * @return The mapping as an array of size numNodes. */ - virtual array1d< globalIndex > getLocalToGlobal() const = 0; + [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; /** * @brief Returns the node sets. Key of the map is the name of the set. * @return A reference to constant map. */ - virtual std::map< string, SortedArray< localIndex > > const & getNodeSets() const = 0; + [[nodiscard]] virtual std::map< string, SortedArray< localIndex > > const & getNodeSets() const = 0; }; } From 04e23ff6aa361d284ee9c315391d0954520c1199 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 19:00:43 -0700 Subject: [PATCH 055/106] Save commit before removing lots of dead code. --- .../mesh/generators/NewGhosting.cpp | 197 +++++++++++++++--- 1 file changed, 173 insertions(+), 24 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index c356c7f485b..7853d3f3490 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -23,12 +23,14 @@ #include "common/MpiWrapper.hpp" #include "common/DataTypes.hpp" +#include +#include +#include #include #include #include -#include -#include #include +#include #include "Indices.hpp" @@ -837,6 +839,25 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su return result; } +std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, + bool makeDataContiguous = true ) +{ + Epetra_RowMatrixTransposer transposer( &input ); // This process does not modify the original `ghosted` matrix. + Epetra_CrsMatrix * tr = nullptr; // The transposer returns a pointer we must handle ourselves. + transposer.CreateTranspose( makeDataContiguous, tr ); + + std::unique_ptr< Epetra_CrsMatrix > ptr; + ptr.reset( tr ); + return ptr; +} + +/** + * Contains the full result of the ghosting + */ +struct Ghost +{ + std::map< EdgeGlbIdx, MpiRank > edges; +}; void assembleAdjacencyMatrix( MeshGraph const & graph, MaxGlbIdcs const & gis, @@ -953,9 +974,11 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, adj.FillComplete(); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/adj.mat", adj ); + int const commSize( MpiWrapper::commSize() ); + // Now let's build the domain indicator matrix. // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. - Epetra_Map const mpiMap( MpiWrapper::commSize(), 0, comm ); // Rows + Epetra_Map const mpiMap( commSize, 0, comm ); // Let the current rank get the appropriate index in this map. Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, mpiMap, numOwned + numOther, true ); std::vector< double > const ones( n, 1. ); @@ -965,47 +988,67 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_Map const graphNodeMap( int( n ), 0, comm ); // Columns indicator.FillComplete( graphNodeMap, mpiMap ); +// Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, mpiMap, numOwned, true ); +// ownership.InsertGlobalValues( curRank.get(), numOwned, ones.data(), ownedGlbIdcs.data() ); +// ownership.FillComplete( graphNodeMap, mpiMap ); +// std::vector< double > myRank( numOwned, curRank.get() ); +// Epetra_Vector ownership( Epetra_DataAccess::Copy, rowMap, myRank.data() ); +// EpetraExt::VectorToMatrixMarketFile( "/tmp/matrices/ownership.mat", ownership ); + std::vector< double > myRank( 1, curRank.get() ); + Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, rowMap, 1, true ); + for( auto const & i: ownedGlbIdcs ) + { + ownership.InsertGlobalValues( i, 1, myRank.data(), &i ); + } + ownership.FillComplete( graphNodeMap, graphNodeMap ); +// ownership.FillComplete( rowMap, rowMap ); +// ownership.FillComplete(); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ownership.mat", ownership ); + if( curRank == 0_mpi ) { GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); +// GEOS_LOG_RANK( "ownership.GlobalLength() = " << ownership.GlobalLength() ); + GEOS_LOG_RANK( "ownership.NumGlobalCols() = " << ownership.NumGlobalCols() ); + GEOS_LOG_RANK( "ownership.NumGlobalRows() = " << ownership.NumGlobalRows() ); + GEOS_LOG_RANK( "ownership diag = " << std::boolalpha << ownership.LowerTriangular() and ownership.UpperTriangular() ); } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); - int NN( MpiWrapper::commSize() ); GEOS_LOG_RANK("A"); auto multiply = [&]()-> Epetra_CrsMatrix { // Upward (n -> e -> f -> c) - Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, rowMap, NN, false ); + Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, rowMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result_u0_0, false ); result_u0_0.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); - Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, rowMap, NN, false ); + Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, rowMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_0, false, result_u0_1, false ); result_u0_1.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); - Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, rowMap, NN, false ); + Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, rowMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_1, false, result_u0_2, false ); result_u0_2.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); // Downward (c -> f -> e -> n) - Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, rowMap, NN, false ); + Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, rowMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result_u0_2, false, result_d0_0, false ); result_d0_0.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); - Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, rowMap, NN, false ); + Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, rowMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_0, false, result_d0_1, false ); result_d0_1.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); - Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, rowMap, NN, false ); + Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, rowMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_1, false, result_d0_2, false ); result_d0_2.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); @@ -1014,15 +1057,42 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, }; Epetra_CrsMatrix ghosted( multiply() ); + ghosted.PutScalar( 1. ); // This can be done after the `FillComplete`. ghosted.FillComplete( mpiMap, graphNodeMap ); +// ghosted.FillComplete( mpiMap, rowMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); - // Transposing in order to get easy access to the row... - Epetra_RowMatrixTransposer transposer( &ghosted ); - Epetra_CrsMatrix * ptGhosted = nullptr; // The transposer returns a pointer we must handle ourselves. - transposer.CreateTranspose( true, ptGhosted ); - GEOS_LOG_RANK( "ptGhosted->NumGlobalCols() = " << ptGhosted->NumGlobalCols() ); - GEOS_LOG_RANK( "ptGhosted->NumGlobalRows() = " << ptGhosted->NumGlobalRows() ); + std::unique_ptr< Epetra_CrsMatrix > ptGhosted = makeTranspose( ghosted ); + if( curRank == 0_mpi ) + { + GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghosted.NumGlobalCols() ); + GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghosted.NumGlobalRows() ); + GEOS_LOG_RANK( "ptGhosted->NumGlobalCols() = " << ptGhosted->NumGlobalCols() ); + GEOS_LOG_RANK( "ptGhosted->NumGlobalRows() = " << ptGhosted->NumGlobalRows() ); + } + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted-after.mat", ghosted ); + + Epetra_CrsMatrix ghostInfo( Epetra_DataAccess::Copy, mpiMap, 1, false ); + EpetraExt::MatrixMatrix::Multiply( ghosted, true, ownership, false, ghostInfo, false ); +// ghostInfo.FillComplete( rowMap, mpiMap ); + ghostInfo.FillComplete( graphNodeMap, mpiMap ); +// ghostInfo.FillComplete(); + if( curRank == 0_mpi ) + { + GEOS_LOG_RANK( "ghostInfo->NumGlobalCols() = " << ghostInfo.NumGlobalCols() ); + GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostInfo.NumGlobalRows() ); + } + + int extracted3 = 0; + std::vector< double > extractedValues3( n ); + std::vector< int > extractedIndices3( n ); + ghostInfo.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted3, extractedValues3.data(), extractedIndices3.data() ); + extractedValues3.resize( extracted3 ); + extractedIndices3.resize( extracted3 ); + GEOS_LOG_RANK( "extracted3 = " << extracted3 ); +// GEOS_LOG_RANK( "extractedValues3 = " << json(extractedValues3) ); +// GEOS_LOG_RANK( "extractedIndices3 = " << json(extractedIndices3) ); + // My test says that `tGhosted` is filled here. int extracted = 0; @@ -1032,27 +1102,106 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, extractedValues.resize( extracted ); extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); + + + std::unique_ptr< Epetra_CrsMatrix > ptIndicator = makeTranspose( indicator ); + if( curRank == 0_mpi ) { - std::vector< int > interest; - for( int i = 0; i < extracted; ++i ) + GEOS_LOG_RANK( "ptIndicator->NumGlobalCols() = " << ptIndicator->NumGlobalCols() ); + GEOS_LOG_RANK( "ptIndicator->NumGlobalRows() = " << ptIndicator->NumGlobalRows() ); + } + + int extracted2 = 0; + std::vector< double > ranksValues( commSize ) ; + std::vector< int > ranksIndices( commSize ) ; + std::map< NodeGlbIdx , MpiRank > nodes; + { +// std::vector< int > interest; +// for( int i = 0; i < extracted; ++i ) + for( int i = 0; i < extracted3; ++i ) { - int const & index = extractedIndices[i]; - double const & val = extractedValues[i]; - if( val > 0 and index >= int( cellOffset ) ) +// int const & index = extractedIndices[i]; +// double const & val = extractedValues[i]; + int const & index = extractedIndices3[i]; + MpiRank const owner = MpiRank{ int( extractedValues3[i] ) }; + if( index < int( edgeOffset ) ) // This is a node to consider { - interest.push_back( index - cellOffset ); + NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; + nodes.emplace( ngi, owner ); } + +// if( !( val > 0 ) ) // Prevents against floating point exact comparison +// { +// continue; +// } +// if( val > 0 and int( edgeOffset ) <= index and index < int( faceOffset ) ) // This is an edge to consider + +// if( index < int( edgeOffset ) ) // This is a node to consider +// { +// NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; +// MpiRank owner = curRank; // In case we own. +// if( graph.nodes.find( ngi ) == graph.nodes.cend() ) +// { +// // TODO maybe I need the transposed of indicator? +// // We do not own the edge, so we have to get the actual owner +// ptIndicator->ExtractGlobalRowCopy( index, commSize, extracted2, ranksValues.data(), ranksIndices.data() ); +// std::vector< int > tmp; +// if(extracted2 == 0) +// { +//// GEOS_LOG_RANK("extracted2 = " << extracted2 ); +// continue; +// } +// for( auto ii = 0; ii < extracted2; ++ii ) +// { +// if( ranksValues[ii] > 0 ) +// { +// tmp.emplace_back( ranksIndices[ii] ); +// } +// } +//// auto tmp = Span< int >( ranksIndices.data(), extracted2 ); +// owner = MpiRank{ *std::min_element( std::cbegin( tmp ), std::cend( tmp ) ) }; // TODO check that there's at least one element or die. +// } +// nodes.emplace( ngi, owner ); + + // interest.push_back( index - cellOffset ); +// } // if( val > 0 and index < int( edgeOffset ) ) // { // interest.push_back( index ); // } } - GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); +// GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); } +// GEOS_LOG_RANK( "ghost nodes = " << json( nodes ) ); + + if( curRank == 2_mpi ) + { + std::map< MpiRank, std::set< NodeGlbIdx > > tmp; + for( auto const & [node, rk]: nodes ) + { + tmp[rk].insert( node ); + } + for( auto const & [rk, nn]: tmp ) + { + GEOS_LOG_RANK( "ghost nodes = " << rk << ": " << json( nn ) ); + } + } +// { +// std::vector< NodeGlbIdx > interest; +// for( auto const & [ngi, _]: nodes ) +// { +// interest.emplace_back( ngi ); +// } +// GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); +// } + + // TODO to build the edges -> nodes map containing the all the ghost (i.e. the ghosted ones as well), + // Let's create a vector (or matrix?) full of ones where we have edges and multiply using the adjacency matrix. + GEOS_LOG_RANK("B"); - delete ptGhosted; +// delete ptGhosted; } From ec259425cb8b341d04189ab671d73ec91dd313d8 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 20:50:04 -0700 Subject: [PATCH 056/106] remove dead code --- .../mesh/generators/NewGhosting.cpp | 105 +----------------- 1 file changed, 5 insertions(+), 100 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 7853d3f3490..0b511ed9540 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -988,12 +988,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_Map const graphNodeMap( int( n ), 0, comm ); // Columns indicator.FillComplete( graphNodeMap, mpiMap ); -// Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, mpiMap, numOwned, true ); -// ownership.InsertGlobalValues( curRank.get(), numOwned, ones.data(), ownedGlbIdcs.data() ); -// ownership.FillComplete( graphNodeMap, mpiMap ); -// std::vector< double > myRank( numOwned, curRank.get() ); -// Epetra_Vector ownership( Epetra_DataAccess::Copy, rowMap, myRank.data() ); -// EpetraExt::VectorToMatrixMarketFile( "/tmp/matrices/ownership.mat", ownership ); + // TODO Could we use an Epetra_Vector as a diagonal matrix? std::vector< double > myRank( 1, curRank.get() ); Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, rowMap, 1, true ); for( auto const & i: ownedGlbIdcs ) @@ -1001,15 +996,12 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, ownership.InsertGlobalValues( i, 1, myRank.data(), &i ); } ownership.FillComplete( graphNodeMap, graphNodeMap ); -// ownership.FillComplete( rowMap, rowMap ); -// ownership.FillComplete(); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ownership.mat", ownership ); if( curRank == 0_mpi ) { GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); -// GEOS_LOG_RANK( "ownership.GlobalLength() = " << ownership.GlobalLength() ); GEOS_LOG_RANK( "ownership.NumGlobalCols() = " << ownership.NumGlobalCols() ); GEOS_LOG_RANK( "ownership.NumGlobalRows() = " << ownership.NumGlobalRows() ); GEOS_LOG_RANK( "ownership diag = " << std::boolalpha << ownership.LowerTriangular() and ownership.UpperTriangular() ); @@ -1059,122 +1051,46 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_CrsMatrix ghosted( multiply() ); ghosted.PutScalar( 1. ); // This can be done after the `FillComplete`. ghosted.FillComplete( mpiMap, graphNodeMap ); -// ghosted.FillComplete( mpiMap, rowMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); - std::unique_ptr< Epetra_CrsMatrix > ptGhosted = makeTranspose( ghosted ); if( curRank == 0_mpi ) { GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghosted.NumGlobalCols() ); GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghosted.NumGlobalRows() ); - GEOS_LOG_RANK( "ptGhosted->NumGlobalCols() = " << ptGhosted->NumGlobalCols() ); - GEOS_LOG_RANK( "ptGhosted->NumGlobalRows() = " << ptGhosted->NumGlobalRows() ); } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted-after.mat", ghosted ); Epetra_CrsMatrix ghostInfo( Epetra_DataAccess::Copy, mpiMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( ghosted, true, ownership, false, ghostInfo, false ); -// ghostInfo.FillComplete( rowMap, mpiMap ); ghostInfo.FillComplete( graphNodeMap, mpiMap ); -// ghostInfo.FillComplete(); if( curRank == 0_mpi ) { GEOS_LOG_RANK( "ghostInfo->NumGlobalCols() = " << ghostInfo.NumGlobalCols() ); GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostInfo.NumGlobalRows() ); } - int extracted3 = 0; - std::vector< double > extractedValues3( n ); - std::vector< int > extractedIndices3( n ); - ghostInfo.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted3, extractedValues3.data(), extractedIndices3.data() ); - extractedValues3.resize( extracted3 ); - extractedIndices3.resize( extracted3 ); - GEOS_LOG_RANK( "extracted3 = " << extracted3 ); -// GEOS_LOG_RANK( "extractedValues3 = " << json(extractedValues3) ); -// GEOS_LOG_RANK( "extractedIndices3 = " << json(extractedIndices3) ); - - - // My test says that `tGhosted` is filled here. int extracted = 0; std::vector< double > extractedValues( n ); std::vector< int > extractedIndices( n ); - ptGhosted->ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); + ghostInfo.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); extractedValues.resize( extracted ); extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); - - std::unique_ptr< Epetra_CrsMatrix > ptIndicator = makeTranspose( indicator ); - if( curRank == 0_mpi ) - { - GEOS_LOG_RANK( "ptIndicator->NumGlobalCols() = " << ptIndicator->NumGlobalCols() ); - GEOS_LOG_RANK( "ptIndicator->NumGlobalRows() = " << ptIndicator->NumGlobalRows() ); - } - - int extracted2 = 0; - std::vector< double > ranksValues( commSize ) ; - std::vector< int > ranksIndices( commSize ) ; std::map< NodeGlbIdx , MpiRank > nodes; { -// std::vector< int > interest; -// for( int i = 0; i < extracted; ++i ) - for( int i = 0; i < extracted3; ++i ) + for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? { -// int const & index = extractedIndices[i]; -// double const & val = extractedValues[i]; - int const & index = extractedIndices3[i]; - MpiRank const owner = MpiRank{ int( extractedValues3[i] ) }; + int const & index = extractedIndices[i]; + MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; if( index < int( edgeOffset ) ) // This is a node to consider { NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; nodes.emplace( ngi, owner ); } - -// if( !( val > 0 ) ) // Prevents against floating point exact comparison -// { -// continue; -// } -// if( val > 0 and int( edgeOffset ) <= index and index < int( faceOffset ) ) // This is an edge to consider - -// if( index < int( edgeOffset ) ) // This is a node to consider -// { -// NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; -// MpiRank owner = curRank; // In case we own. -// if( graph.nodes.find( ngi ) == graph.nodes.cend() ) -// { -// // TODO maybe I need the transposed of indicator? -// // We do not own the edge, so we have to get the actual owner -// ptIndicator->ExtractGlobalRowCopy( index, commSize, extracted2, ranksValues.data(), ranksIndices.data() ); -// std::vector< int > tmp; -// if(extracted2 == 0) -// { -//// GEOS_LOG_RANK("extracted2 = " << extracted2 ); -// continue; -// } -// for( auto ii = 0; ii < extracted2; ++ii ) -// { -// if( ranksValues[ii] > 0 ) -// { -// tmp.emplace_back( ranksIndices[ii] ); -// } -// } -//// auto tmp = Span< int >( ranksIndices.data(), extracted2 ); -// owner = MpiRank{ *std::min_element( std::cbegin( tmp ), std::cend( tmp ) ) }; // TODO check that there's at least one element or die. -// } -// nodes.emplace( ngi, owner ); - - // interest.push_back( index - cellOffset ); -// } -// if( val > 0 and index < int( edgeOffset ) ) -// { -// interest.push_back( index ); -// } } -// GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); } -// GEOS_LOG_RANK( "ghost nodes = " << json( nodes ) ); - if( curRank == 2_mpi ) { std::map< MpiRank, std::set< NodeGlbIdx > > tmp; @@ -1187,21 +1103,10 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_LOG_RANK( "ghost nodes = " << rk << ": " << json( nn ) ); } } -// { -// std::vector< NodeGlbIdx > interest; -// for( auto const & [ngi, _]: nodes ) -// { -// interest.emplace_back( ngi ); -// } -// GEOS_LOG_RANK( "ghost interest = " << json( interest ) ); -// } // TODO to build the edges -> nodes map containing the all the ghost (i.e. the ghosted ones as well), // Let's create a vector (or matrix?) full of ones where we have edges and multiply using the adjacency matrix. - GEOS_LOG_RANK("B"); - -// delete ptGhosted; } From 08cf210f91629e6c74b3daf88536af9d84d7050b Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 20:57:36 -0700 Subject: [PATCH 057/106] Changed rowMap into ownedMap --- .../mesh/generators/NewGhosting.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 0b511ed9540..c8729dee598 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -959,9 +959,9 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // GEOS_LOG_RANK( "ownedGlbIdcs = " << json( ownedGlbIdcs ) ); Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); - Epetra_Map const rowMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); + Epetra_Map const ownedMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); - Epetra_CrsMatrix adj( Epetra_DataAccess::Copy, rowMap, numEntriesPerRow.data(), true ); + Epetra_CrsMatrix adj( Epetra_DataAccess::Copy, ownedMap, numEntriesPerRow.data(), true ); for( std::size_t i = 0; i < numOwned; ++i ) { @@ -990,7 +990,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, // TODO Could we use an Epetra_Vector as a diagonal matrix? std::vector< double > myRank( 1, curRank.get() ); - Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, rowMap, 1, true ); + Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, ownedMap, 1, true ); for( auto const & i: ownedGlbIdcs ) { ownership.InsertGlobalValues( i, 1, myRank.data(), &i ); @@ -1013,34 +1013,34 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, { // Upward (n -> e -> f -> c) - Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, rowMap, commSize, false ); + Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result_u0_0, false ); result_u0_0.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); - Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, rowMap, commSize, false ); + Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_0, false, result_u0_1, false ); result_u0_1.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); - Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, rowMap, commSize, false ); + Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_1, false, result_u0_2, false ); result_u0_2.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); // Downward (c -> f -> e -> n) - Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, rowMap, commSize, false ); + Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result_u0_2, false, result_d0_0, false ); result_d0_0.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); - Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, rowMap, commSize, false ); + Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_0, false, result_d0_1, false ); result_d0_1.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); - Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, rowMap, commSize, false ); + Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_1, false, result_d0_2, false ); result_d0_2.FillComplete( mpiMap, graphNodeMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); From 857db9f149988e14bbb3f0d4ed7d3cb05b25b1e2 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 15 May 2024 21:05:12 -0700 Subject: [PATCH 058/106] Add [[nodiscard]]s --- src/coreComponents/mesh/generators/Pods.hpp | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 2a2f89573f2..d9fc315b307 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -30,19 +30,19 @@ class NodeMgrImpl : public generators::NodeMgr public: explicit NodeMgrImpl( NodeLocIdx const & numNodes ); - localIndex numNodes() const override; + [[nodiscard]] localIndex numNodes() const override; - array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override; + [[nodiscard]] array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override; - ArrayOfArrays< localIndex > getNodeToEdges() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getNodeToEdges() const override; - ArrayOfArrays< localIndex > getNodeToFaces() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getNodeToFaces() const override; - ToCellRelation< ArrayOfArrays< localIndex>> getNodeToElements() const override; + [[nodiscard]] ToCellRelation< ArrayOfArrays< localIndex>> getNodeToElements() const override; - array1d< globalIndex > getLocalToGlobal() const override; + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; - std::map< string, SortedArray< localIndex > > const & getNodeSets() const override; + [[nodiscard]] std::map< string, SortedArray< localIndex > > const & getNodeSets() const override; private: NodeLocIdx m_numNodes; @@ -54,13 +54,13 @@ class EdgeMgrImpl : public generators::EdgeMgr public: explicit EdgeMgrImpl( EdgeLocIdx const & numEdges ); - localIndex numEdges() const override; + [[nodiscard]] localIndex numEdges() const override; - array2d< localIndex > getEdgeToNodes() const override; + [[nodiscard]] array2d< localIndex > getEdgeToNodes() const override; - ArrayOfArrays< localIndex > getEdgeToFaces() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getEdgeToFaces() const override; - array1d< integer > getGhostRank() const override; + [[nodiscard]] array1d< integer > getGhostRank() const override; private: EdgeLocIdx m_numEdges; @@ -71,13 +71,13 @@ class FaceMgrImpl : public generators::FaceMgr public: explicit FaceMgrImpl( FaceLocIdx const & numFaces ); - localIndex numFaces() const override; + [[nodiscard]] localIndex numFaces() const override; - ArrayOfArrays< localIndex > getFaceToNodes() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getFaceToNodes() const override; - ArrayOfArrays< localIndex > getFaceToEdges() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getFaceToEdges() const override; - ToCellRelation< array2d< localIndex > > getFaceToElements() const override; + [[nodiscard]] ToCellRelation< array2d< localIndex > > getFaceToElements() const override; private: FaceLocIdx m_numFaces; From 4113721a7b69af8e0b9e3d92243d1402107995d1 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 16 May 2024 13:42:41 -0700 Subject: [PATCH 059/106] Remove graphNodeMap and compute the transposed matrix once for all. --- .../mesh/generators/NewGhosting.cpp | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index c8729dee598..0eb0f5258f4 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -984,9 +984,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, std::vector< double > const ones( n, 1. ); indicator.InsertGlobalValues( curRank.get(), numOwned, ones.data(), ownedGlbIdcs.data() ); indicator.InsertGlobalValues( curRank.get(), numOther, ones.data(), otherGlbIdcs.data() ); - - Epetra_Map const graphNodeMap( int( n ), 0, comm ); // Columns - indicator.FillComplete( graphNodeMap, mpiMap ); + indicator.FillComplete( ownedMap, mpiMap ); // TODO Could we use an Epetra_Vector as a diagonal matrix? std::vector< double > myRank( 1, curRank.get() ); @@ -995,7 +993,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, { ownership.InsertGlobalValues( i, 1, myRank.data(), &i ); } - ownership.FillComplete( graphNodeMap, graphNodeMap ); + ownership.FillComplete( ownedMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ownership.mat", ownership ); if( curRank == 0_mpi ) @@ -1015,34 +1013,35 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result_u0_0, false ); - result_u0_0.FillComplete( mpiMap, graphNodeMap ); + result_u0_0.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_0, false, result_u0_1, false ); - result_u0_1.FillComplete( mpiMap, graphNodeMap ); + result_u0_1.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_1, false, result_u0_2, false ); - result_u0_2.FillComplete( mpiMap, graphNodeMap ); + result_u0_2.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); // Downward (c -> f -> e -> n) + auto tAdj = makeTranspose( adj ); // TODO check the algorithm to understand what's more relevant. Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result_u0_2, false, result_d0_0, false ); - result_d0_0.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::MatrixMatrix::Multiply( *tAdj, false, result_u0_2, false, result_d0_0, false ); + result_d0_0.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_0, false, result_d0_1, false ); - result_d0_1.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::MatrixMatrix::Multiply( *tAdj, false, result_d0_0, false, result_d0_1, false ); + result_d0_1.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( adj, true, result_d0_1, false, result_d0_2, false ); - result_d0_2.FillComplete( mpiMap, graphNodeMap ); + EpetraExt::MatrixMatrix::Multiply( *tAdj, false, result_d0_1, false, result_d0_2, false ); + result_d0_2.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); return result_d0_2; @@ -1050,7 +1049,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_CrsMatrix ghosted( multiply() ); ghosted.PutScalar( 1. ); // This can be done after the `FillComplete`. - ghosted.FillComplete( mpiMap, graphNodeMap ); + ghosted.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); if( curRank == 0_mpi ) @@ -1062,7 +1061,7 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_CrsMatrix ghostInfo( Epetra_DataAccess::Copy, mpiMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( ghosted, true, ownership, false, ghostInfo, false ); - ghostInfo.FillComplete( graphNodeMap, mpiMap ); + ghostInfo.FillComplete( ownedMap, mpiMap ); if( curRank == 0_mpi ) { GEOS_LOG_RANK( "ghostInfo->NumGlobalCols() = " << ghostInfo.NumGlobalCols() ); From 8273d6d186e049cc9b757b238775f77838959e34 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 16 May 2024 14:25:31 -0700 Subject: [PATCH 060/106] Extract neighbor data for all the geometrical quantities. --- .../mesh/generators/NewGhosting.cpp | 56 ++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 0eb0f5258f4..fcb0b470b98 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -955,9 +955,6 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_ASSERT_EQ( indices[i].size(), std::size_t( numEntriesPerRow[i] ) ); } -// GEOS_LOG_RANK( "n = " << n ); -// GEOS_LOG_RANK( "ownedGlbIdcs = " << json( ownedGlbIdcs ) ); - Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); Epetra_Map const ownedMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); @@ -1006,7 +1003,6 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); - GEOS_LOG_RANK("A"); auto multiply = [&]()-> Epetra_CrsMatrix { // Upward (n -> e -> f -> c) @@ -1077,35 +1073,55 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_LOG_RANK( "extracted = " << extracted ); std::map< NodeGlbIdx , MpiRank > nodes; + std::map< EdgeGlbIdx , MpiRank > edges; + std::map< FaceGlbIdx , MpiRank > faces; + std::map< CellGlbIdx , MpiRank > cells; + std::set< MpiRank > neighbors; { for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? { int const & index = extractedIndices[i]; MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; - if( index < int( edgeOffset ) ) // This is a node to consider + neighbors.insert( owner ); + if( index < int( edgeOffset ) ) // This is a node { NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; nodes.emplace( ngi, owner ); } + else if( int( edgeOffset ) <= index && index < int( faceOffset ) ) + { + EdgeGlbIdx const egi = EdgeGlbIdx{ intConv< globalIndex >( index - edgeOffset ) }; + edges.emplace( egi, owner ); + } + else if( int( faceOffset ) <= index && index < int( cellOffset ) ) + { + FaceGlbIdx const fgi = FaceGlbIdx{ intConv< globalIndex >( index - faceOffset ) }; + faces.emplace( fgi, owner ); + } + else + { + CellGlbIdx const cgi = CellGlbIdx{ intConv< globalIndex >( index - cellOffset ) }; + cells.emplace( cgi, owner ); + } } } - if( curRank == 2_mpi ) - { - std::map< MpiRank, std::set< NodeGlbIdx > > tmp; - for( auto const & [node, rk]: nodes ) - { - tmp[rk].insert( node ); - } - for( auto const & [rk, nn]: tmp ) - { - GEOS_LOG_RANK( "ghost nodes = " << rk << ": " << json( nn ) ); - } - } +// if( curRank == 2_mpi ) +// { +// std::map< MpiRank, std::set< CellGlbIdx > > tmp; +// for( auto const & [geom, rk]: cells ) +// { +// tmp[rk].insert( geom ); +// } +// for( auto const & [rk, geom]: tmp ) +// { +// GEOS_LOG_RANK( "ghost geom = " << rk << ": " << json( geom ) ); +// } +// } +// GEOS_LOG_RANK( "my final neighbors are " << json( neighbors ) ); // TODO to build the edges -> nodes map containing the all the ghost (i.e. the ghosted ones as well), // Let's create a vector (or matrix?) full of ones where we have edges and multiply using the adjacency matrix. - GEOS_LOG_RANK("B"); } @@ -1151,7 +1167,8 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v { // For the other ranks, the reduction operator will be called during the `Mpi_Scan` process. // So unlike for rank 0, we do not have to do it ourselves. - // In order to provide the `sizes` to the reduction operator, since `sizes` will only be used on the current ranl, + // In order to provide the `sizes` to the reduction operator, + // since `sizes` will only be used on the current rank, // we'll provide the information as a pointer to the instance. // The reduction operator will then compute the new offsets and send them to the following rank. std::uintptr_t const addr = reinterpret_cast(&sizes); @@ -1197,6 +1214,7 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v { neighbors_.insert( MpiRank{ rank } ); } + GEOS_LOG_RANK( "my initial neighbors are " << json( neighbors_ ) ); return doTheNewGhosting( mesh, neighbors_ ); } From c6528adec990e9b36e13a03a1ad69ba7e95f24bd Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 16 May 2024 14:32:44 -0700 Subject: [PATCH 061/106] Use a compound struct. --- .../mesh/generators/NewGhosting.cpp | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index fcb0b470b98..2cc26082a5b 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -856,12 +856,16 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, */ struct Ghost { - std::map< EdgeGlbIdx, MpiRank > edges; + std::map< NodeGlbIdx , MpiRank > nodes; + std::map< EdgeGlbIdx , MpiRank > edges; + std::map< FaceGlbIdx , MpiRank > faces; + std::map< CellGlbIdx , MpiRank > cells; + std::set< MpiRank > neighbors; }; -void assembleAdjacencyMatrix( MeshGraph const & graph, - MaxGlbIdcs const & gis, - MpiRank curRank ) +Ghost assembleAdjacencyMatrix( MeshGraph const & graph, + MaxGlbIdcs const & gis, + MpiRank curRank ) { std::size_t const edgeOffset = gis.nodes.get() + 1; std::size_t const faceOffset = edgeOffset + gis.edges.get() + 1; @@ -1072,40 +1076,38 @@ void assembleAdjacencyMatrix( MeshGraph const & graph, extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); - std::map< NodeGlbIdx , MpiRank > nodes; - std::map< EdgeGlbIdx , MpiRank > edges; - std::map< FaceGlbIdx , MpiRank > faces; - std::map< CellGlbIdx , MpiRank > cells; - std::set< MpiRank > neighbors; + Ghost ghost; { for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? { int const & index = extractedIndices[i]; MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; - neighbors.insert( owner ); + ghost.neighbors.insert( owner ); if( index < int( edgeOffset ) ) // This is a node { NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; - nodes.emplace( ngi, owner ); + ghost.nodes.emplace( ngi, owner ); } else if( int( edgeOffset ) <= index && index < int( faceOffset ) ) { EdgeGlbIdx const egi = EdgeGlbIdx{ intConv< globalIndex >( index - edgeOffset ) }; - edges.emplace( egi, owner ); + ghost.edges.emplace( egi, owner ); } else if( int( faceOffset ) <= index && index < int( cellOffset ) ) { FaceGlbIdx const fgi = FaceGlbIdx{ intConv< globalIndex >( index - faceOffset ) }; - faces.emplace( fgi, owner ); + ghost.faces.emplace( fgi, owner ); } else { CellGlbIdx const cgi = CellGlbIdx{ intConv< globalIndex >( index - cellOffset ) }; - cells.emplace( cgi, owner ); + ghost.cells.emplace( cgi, owner ); } } } + return ghost; + // if( curRank == 2_mpi ) // { // std::map< MpiRank, std::set< CellGlbIdx > > tmp; @@ -1197,11 +1199,11 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v // std::cout << "My graph is " << json( graph ) << std::endl; // } - assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); + Ghost const ghost = assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); - NodeMgrImpl const nodeMgr( 0_nli ); - EdgeMgrImpl const edgeMgr( 0_eli ); - FaceMgrImpl const faceMgr( 0_fli ); + NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( std::size( ghost.nodes ) ) } ); + EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( std::size( ghost.edges ) ) } ); + FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( std::size( ghost.faces ) ) } ); return {}; } From ec7ea67a0df677fd170fb1a283563daa4bdd1e2d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 16 May 2024 16:52:39 -0700 Subject: [PATCH 062/106] Create ghostRank and l2g for EdgeMgr. --- .../mesh/generators/NewGhosting.cpp | 87 +++++++++++++------ src/coreComponents/mesh/generators/Pods.cpp | 27 +++++- src/coreComponents/mesh/generators/Pods.hpp | 8 +- .../mesh/generators/include/EdgeMgr.hpp | 3 +- 4 files changed, 93 insertions(+), 32 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 2cc26082a5b..916184c5d58 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1077,32 +1077,30 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_LOG_RANK( "extracted = " << extracted ); Ghost ghost; + for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? { - for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? + int const & index = extractedIndices[i]; + MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; + ghost.neighbors.insert( owner ); + if( index < int( edgeOffset ) ) // This is a node { - int const & index = extractedIndices[i]; - MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; - ghost.neighbors.insert( owner ); - if( index < int( edgeOffset ) ) // This is a node - { - NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; - ghost.nodes.emplace( ngi, owner ); - } - else if( int( edgeOffset ) <= index && index < int( faceOffset ) ) - { - EdgeGlbIdx const egi = EdgeGlbIdx{ intConv< globalIndex >( index - edgeOffset ) }; - ghost.edges.emplace( egi, owner ); - } - else if( int( faceOffset ) <= index && index < int( cellOffset ) ) - { - FaceGlbIdx const fgi = FaceGlbIdx{ intConv< globalIndex >( index - faceOffset ) }; - ghost.faces.emplace( fgi, owner ); - } - else - { - CellGlbIdx const cgi = CellGlbIdx{ intConv< globalIndex >( index - cellOffset ) }; - ghost.cells.emplace( cgi, owner ); - } + NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; + ghost.nodes.emplace( ngi, owner ); + } + else if( int( edgeOffset ) <= index && index < int( faceOffset ) ) + { + EdgeGlbIdx const egi = EdgeGlbIdx{ intConv< globalIndex >( index - edgeOffset ) }; + ghost.edges.emplace( egi, owner ); + } + else if( int( faceOffset ) <= index && index < int( cellOffset ) ) + { + FaceGlbIdx const fgi = FaceGlbIdx{ intConv< globalIndex >( index - faceOffset ) }; + ghost.faces.emplace( fgi, owner ); + } + else + { + CellGlbIdx const cgi = CellGlbIdx{ intConv< globalIndex >( index - cellOffset ) }; + ghost.cells.emplace( cgi, owner ); } } @@ -1126,6 +1124,43 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, // Let's create a vector (or matrix?) full of ones where we have edges and multiply using the adjacency matrix. } +/** + * @brief + * @tparam GI A global index type + * @param m + * @return + */ +template< class GI > +std::tuple< std::vector< MpiRank >, std::vector< GI > > +buildGhostRankAndL2G( std::map< GI, MpiRank > const & m ) +{ + std::size_t const size = std::size( m ); + + std::vector< MpiRank > ghostRank; + ghostRank.reserve( size ); + std::vector< GI > l2g; + l2g.reserve( size ); + for( auto const & [t, rank]: m ) + { + ghostRank.emplace_back( rank ); + l2g.emplace_back( t ); + } + + return std::make_tuple( ghostRank, l2g ); +} + +void buildPods( Ghost const & ghost ) +{ + std::size_t const numNodes = std::size( ghost.nodes ); + std::size_t const numEdges = std::size( ghost.edges ); + std::size_t const numFaces = std::size( ghost.faces ); + + auto [ghostRank, l2g] = buildGhostRankAndL2G( ghost.edges ); + + NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); + EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); + FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( numFaces ) } ); +} std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) @@ -1201,9 +1236,7 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v Ghost const ghost = assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); - NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( std::size( ghost.nodes ) ) } ); - EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( std::size( ghost.edges ) ) } ); - FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( std::size( ghost.faces ) ) } ); + buildPods( ghost ); return {}; } diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 24ee9f0d274..61e434f8475 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -56,8 +56,12 @@ std::map< string, SortedArray< localIndex > > const & NodeMgrImpl::getNodeSets() return m_todo; } -EdgeMgrImpl::EdgeMgrImpl( EdgeLocIdx const & numEdges ) - : m_numEdges( numEdges ) +EdgeMgrImpl::EdgeMgrImpl( EdgeLocIdx const & numEdges, + std::vector< MpiRank > && ghostRank, + std::vector< EdgeGlbIdx > && l2g ) + : m_numEdges( numEdges ), + m_ghostRank( ghostRank ), + m_l2g( l2g ) { } localIndex EdgeMgrImpl::numEdges() const @@ -77,7 +81,24 @@ ArrayOfArrays< localIndex > EdgeMgrImpl::getEdgeToFaces() const array1d< integer > EdgeMgrImpl::getGhostRank() const { - return {}; + array1d< integer > result; + result.reserve( std::size( m_ghostRank ) ); + for( MpiRank const & rank: m_ghostRank ) + { + result.emplace_back( rank.get() ); + } + return result; +} + +array1d< globalIndex > EdgeMgrImpl::getLocalToGlobal() const +{ + array1d< globalIndex > result; + result.reserve( std::size( m_l2g ) ); + for( EdgeGlbIdx const & edge: m_l2g ) + { + result.emplace_back( edge.get() ); + } + return result; } FaceMgrImpl::FaceMgrImpl( FaceLocIdx const & numFaces ) diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index d9fc315b307..da867955606 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -52,7 +52,9 @@ class NodeMgrImpl : public generators::NodeMgr class EdgeMgrImpl : public generators::EdgeMgr { public: - explicit EdgeMgrImpl( EdgeLocIdx const & numEdges ); + EdgeMgrImpl( EdgeLocIdx const & numEdges, + std::vector< MpiRank > && ghostRank, + std::vector< EdgeGlbIdx > && l2g ); [[nodiscard]] localIndex numEdges() const override; @@ -62,8 +64,12 @@ class EdgeMgrImpl : public generators::EdgeMgr [[nodiscard]] array1d< integer > getGhostRank() const override; + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; + private: EdgeLocIdx m_numEdges; + std::vector< MpiRank > m_ghostRank; + std::vector< EdgeGlbIdx > m_l2g; }; class FaceMgrImpl : public generators::FaceMgr diff --git a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp index 654525194e9..fc451e74f7f 100644 --- a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp +++ b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp @@ -47,7 +47,8 @@ class EdgeMgr */ [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; - // TODO add the local -> global and the ghost rank. Use inheritance? + // TODO Use inheritance? + [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; }; } From e10d7967c9fbcd0bfb8154370c0973c2df83095f Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 09:05:29 -0700 Subject: [PATCH 063/106] Gathering the information for missing quantities. --- .../mesh/generators/NewGhosting.cpp | 366 ++++++++++++++---- 1 file changed, 290 insertions(+), 76 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 916184c5d58..f31c545ba82 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -856,13 +856,130 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, */ struct Ghost { - std::map< NodeGlbIdx , MpiRank > nodes; - std::map< EdgeGlbIdx , MpiRank > edges; - std::map< FaceGlbIdx , MpiRank > faces; - std::map< CellGlbIdx , MpiRank > cells; + std::map< NodeGlbIdx, MpiRank > nodes; + std::map< EdgeGlbIdx, MpiRank > edges; + std::map< FaceGlbIdx, MpiRank > faces; + std::map< CellGlbIdx, MpiRank > cells; std::set< MpiRank > neighbors; }; + +Epetra_CrsMatrix multiply( int commSize, + Epetra_CrsMatrix const & indicator, + Epetra_CrsMatrix & upward ) +{ + Epetra_Map const & ownedMap = upward.RowMap(); + Epetra_Map const & mpiMap = indicator.RangeMap(); + + // Upward (n -> e -> f -> c) + + Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); + EpetraExt::MatrixMatrix::Multiply( upward, false, indicator, true, result_u0_0, false ); + result_u0_0.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); + + Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); + EpetraExt::MatrixMatrix::Multiply( upward, false, result_u0_0, false, result_u0_1, false ); + result_u0_1.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); + + Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); + EpetraExt::MatrixMatrix::Multiply( upward, false, result_u0_1, false, result_u0_2, false ); + result_u0_2.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); + + // Downward (c -> f -> e -> n) + auto tDownward = makeTranspose( upward ); // TODO check the algorithm to understand what's more relevant. + + Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); + EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_u0_2, false, result_d0_0, false ); + result_d0_0.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); + + Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); + EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_d0_0, false, result_d0_1, false ); + result_d0_1.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); + + Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); + EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_d0_1, false, result_d0_2, false ); + result_d0_2.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); + + return result_d0_2; +} + +class FindGeometricalType +{ +public: + enum Geom + { + NODE, + EDGE, + FACE, + CELL + }; + + FindGeometricalType( std::size_t const edgeOffset, + std::size_t const faceOffset, + std::size_t const cellOffset ) + : m_edgeOffset( intConv< int >( edgeOffset ) ), + m_faceOffset( intConv< int >( faceOffset ) ), + m_cellOffset( intConv< int >( cellOffset ) ) + { } + + Geom operator()( int const & index ) const + { + if( index < m_edgeOffset ) + { + return Geom::NODE; + } + else if( index < m_faceOffset ) + { + return Geom::EDGE; + } + else if( index < m_cellOffset ) + { + return Geom::FACE; + } + else + { + return Geom::CELL; + } + } + + template< typename GLOBAL_INDEX > + [[nodiscard]] GLOBAL_INDEX as( int const & index ) const + { + if constexpr( std::is_same_v< GLOBAL_INDEX, NodeGlbIdx > ) + { + return NodeGlbIdx{ intConv< NodeGlbIdx::UnderlyingType >( index ) }; + } + else if constexpr( std::is_same_v< GLOBAL_INDEX, EdgeGlbIdx > ) + { + return EdgeGlbIdx{ intConv< EdgeGlbIdx::UnderlyingType >( index - m_edgeOffset ) }; + } + else if constexpr( std::is_same_v< GLOBAL_INDEX, FaceGlbIdx > ) + { + return FaceGlbIdx{ intConv< FaceGlbIdx::UnderlyingType >( index - m_faceOffset ) }; + } + else if constexpr( std::is_same_v< GLOBAL_INDEX, CellGlbIdx > ) + { + return CellGlbIdx{ intConv< CellGlbIdx::UnderlyingType >( index - m_cellOffset ) }; + } + else + { + GEOS_ERROR( "Internal Error." ); + } + } + +private: + int const m_edgeOffset; + int const m_faceOffset; + int const m_cellOffset; +}; + + Ghost assembleAdjacencyMatrix( MeshGraph const & graph, MaxGlbIdcs const & gis, MpiRank curRank ) @@ -962,18 +1079,19 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); Epetra_Map const ownedMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); - Epetra_CrsMatrix adj( Epetra_DataAccess::Copy, ownedMap, numEntriesPerRow.data(), true ); + Epetra_CrsMatrix upward( Epetra_DataAccess::Copy, ownedMap, numEntriesPerRow.data(), true ); for( std::size_t i = 0; i < numOwned; ++i ) { std::vector< int > const & rowIndices = indices[i]; std::vector< double > const rowValues( std::size( rowIndices ), 1. ); GEOS_ASSERT_EQ( std::size( rowIndices ), std::size_t( numEntriesPerRow[i] ) ); - adj.InsertGlobalValues( ownedGlbIdcs[i], std::size( rowIndices ), rowValues.data(), rowIndices.data() ); + upward.InsertGlobalValues( ownedGlbIdcs[i], std::size( rowIndices ), rowValues.data(), rowIndices.data() ); } - adj.FillComplete(); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/adj.mat", adj ); +// upward.FillComplete(); + upward.FillComplete( ownedMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/adj.mat", upward ); int const commSize( MpiWrapper::commSize() ); @@ -1003,51 +1121,11 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); GEOS_LOG_RANK( "ownership.NumGlobalCols() = " << ownership.NumGlobalCols() ); GEOS_LOG_RANK( "ownership.NumGlobalRows() = " << ownership.NumGlobalRows() ); - GEOS_LOG_RANK( "ownership diag = " << std::boolalpha << ownership.LowerTriangular() and ownership.UpperTriangular() ); + GEOS_LOG_RANK( "ownership diag = " << std::boolalpha << ownership.LowerTriangular() and ownership.UpperTriangular() ); } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); - auto multiply = [&]()-> Epetra_CrsMatrix - { - // Upward (n -> e -> f -> c) - - Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, indicator, true, result_u0_0, false ); - result_u0_0.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); - - Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_0, false, result_u0_1, false ); - result_u0_1.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); - - Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( adj, false, result_u0_1, false, result_u0_2, false ); - result_u0_2.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); - - // Downward (c -> f -> e -> n) - auto tAdj = makeTranspose( adj ); // TODO check the algorithm to understand what's more relevant. - - Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( *tAdj, false, result_u0_2, false, result_d0_0, false ); - result_d0_0.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); - - Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( *tAdj, false, result_d0_0, false, result_d0_1, false ); - result_d0_1.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); - - Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( *tAdj, false, result_d0_1, false, result_d0_2, false ); - result_d0_2.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); - - return result_d0_2; - }; - - Epetra_CrsMatrix ghosted( multiply() ); + Epetra_CrsMatrix ghosted( multiply( commSize, indicator, upward ) ); ghosted.PutScalar( 1. ); // This can be done after the `FillComplete`. ghosted.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); @@ -1068,6 +1146,19 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostInfo.NumGlobalRows() ); } + Epetra_CrsMatrix identity( Epetra_DataAccess::Copy, ownedMap, 1, true ); + for( int const & i: ownedGlbIdcs ) + { + identity.InsertGlobalValues( i, 1, ones.data(), &i ); + } + identity.FillComplete(); + + auto tDownward = makeTranspose( upward ); // TODO give it to multiply! +// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-before.mat", *tDownward ); + Epetra_Vector const zeros( ownedMap, true ); + tDownward->ReplaceDiagonalValues( zeros ); // TODO directly build zeros in the function call. +// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-after.mat", *tDownward ); + int extracted = 0; std::vector< double > extractedValues( n ); std::vector< int > extractedIndices( n ); @@ -1076,49 +1167,172 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); + std::vector< int > missingIndices; + missingIndices.reserve( n ); + +// int extractedEdges = 0; +// std::vector< double > extractedEdgesValues( n ); +// std::vector< int > extractedEdgesIndices( n ); + FindGeometricalType const getGeomType( edgeOffset, faceOffset, cellOffset ); + using Geom = FindGeometricalType::Geom; + Ghost ghost; for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? { int const & index = extractedIndices[i]; + MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; - ghost.neighbors.insert( owner ); - if( index < int( edgeOffset ) ) // This is a node + if( owner != curRank ) { - NodeGlbIdx const ngi = NodeGlbIdx{ intConv< globalIndex >( index ) }; - ghost.nodes.emplace( ngi, owner ); + ghost.neighbors.insert( owner ); } - else if( int( edgeOffset ) <= index && index < int( faceOffset ) ) + + switch( getGeomType( index ) ) { - EdgeGlbIdx const egi = EdgeGlbIdx{ intConv< globalIndex >( index - edgeOffset ) }; - ghost.edges.emplace( egi, owner ); + case Geom::NODE: + { + NodeGlbIdx const ngi = getGeomType.as< NodeGlbIdx >( index ); + ghost.nodes.emplace( ngi, owner ); + break; + } + case Geom::EDGE: + { + EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); + ghost.edges.emplace( egi, owner ); + if( graph.e2n.find( egi ) == graph.e2n.cend() and graph.otherEdges.find( egi ) == graph.otherEdges.cend() ) + { + missingIndices.emplace_back( index ); + } + break; + } + case Geom::FACE: + { + FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); + ghost.faces.emplace( fgi, owner ); + if( graph.f2e.find( fgi ) == graph.f2e.cend() and graph.otherFaces.find( fgi ) == graph.otherFaces.cend() ) + { + missingIndices.emplace_back( index ); + } + break; + } + case Geom::CELL: + { + CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); + ghost.cells.emplace( cgi, owner ); + if( graph.c2f.find( cgi ) == graph.c2f.cend() ) + { + missingIndices.emplace_back( index ); + } + break; + } + default: + { + GEOS_ERROR( "Internal error" ); + } } - else if( int( faceOffset ) <= index && index < int( cellOffset ) ) + } + + GEOS_LOG( "Gathering the missing mappings." ); + + std::size_t const numMissingIndices = std::size( missingIndices ); + std::size_t const numGlobalMissingIndices = MpiWrapper::sum( numMissingIndices ); + +// Epetra_Map const missingIndicesMap( intConv< long long int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); + Epetra_Map const missingIndicesMap( intConv< int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); + +// std::size_t const offset = MpiWrapper::prefixSum< std::size_t >( numMissingIndices ); + int const offset = *missingIndicesMap.MyGlobalElements(); +// GEOS_LOG_RANK( "numMissingIndices, numGlobalMissingIndices, offset = " << numMissingIndices << ", " << numGlobalMissingIndices << ", " << offset ); + + Epetra_CrsMatrix missing( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); + for( std::size_t i = 0; i < numMissingIndices; ++i ) + { + missing.InsertGlobalValues( offset + i, 1, ones.data(), &missingIndices[i] ); + } + missing.FillComplete( ownedMap, missingIndicesMap ); + + // TODO Don't put ones in downard: reassemble with the ordering (e.g. edges point to a first and last node) + Epetra_CrsMatrix missingMappings( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); + EpetraExt::MatrixMatrix::Multiply( missing, false, *tDownward, true, missingMappings, false ); + missingMappings.FillComplete( ownedMap, missingIndicesMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingMappings ); + + MeshGraph neighboringConnexions; + + int ext = 0; + std::vector< double > extValues( n ); + std::vector< int > extIndices( n ); +// double * extValues = nullptr; +// int * extIndices = nullptr; + + for( int i = 0; i < int( numMissingIndices ); ++i ) + { + missingMappings.ExtractGlobalRowCopy( offset + i, int( n ), ext, extValues.data(), extIndices.data() ); +// missingMappings.ExtractGlobalRowView( intConv< int >( offset + i ), ext, extValues ); +// missingMappings.ExtractGlobalRowView( intConv< int >( offset + i ), ext, extValues, extIndices ); +// Span< double > const s0( extValues, ext ); +// Span< int > const s1( extIndices, ext ); +// GEOS_LOG( "ext 0 = " << std::size( s0 ) ); +// GEOS_LOG( "ext 1 = " << std::size( s1 ) ); +// GEOS_LOG_RANK( "ext = " << ext ); + int const index = missingIndices[i]; + switch( getGeomType( index ) ) { - FaceGlbIdx const fgi = FaceGlbIdx{ intConv< globalIndex >( index - faceOffset ) }; - ghost.faces.emplace( fgi, owner ); + case Geom::EDGE: + { + GEOS_ASSERT_EQ( ext, 2 ); // TODO check that val != 0? + EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); + NodeGlbIdx const ngi0 = getGeomType.as< NodeGlbIdx >( extIndices[0] ); + NodeGlbIdx const ngi1 = getGeomType.as< NodeGlbIdx >( extIndices[1] ); + neighboringConnexions.e2n[egi] = std::minmax( { ngi0, ngi1 } ); + break; + } + case Geom::FACE: + { + GEOS_ASSERT_EQ( ext, 4 ); // TODO temporary check for the dev + FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); + std::set< EdgeGlbIdx > & tmp = neighboringConnexions.f2e[fgi]; + for( int ii = 0; ii < ext; ++ii ) + { + tmp.insert( EdgeGlbIdx{ intConv< globalIndex >( extIndices[ii] - edgeOffset ) } ); + } + break; + } + case Geom::CELL: + { + GEOS_ASSERT_EQ( ext, 6 ); // TODO temporary check for the dev + CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); + std::set< FaceGlbIdx > & tmp = neighboringConnexions.c2f[cgi]; + for( int ii = 0; ii < ext; ++ii ) + { + tmp.insert( FaceGlbIdx{ intConv< globalIndex >( extIndices[ii] - faceOffset ) } ); + } + break; + } + default: + { + GEOS_ERROR( "Internal error." ); + } } - else + } + + if( curRank == 2_mpi ) + { + GEOS_LOG_RANK( "neighboringConnexions = " << json( neighboringConnexions ) ); + std::map< MpiRank, std::set< CellGlbIdx > > tmp; + for( auto const & [geom, rk]: ghost.cells ) + { + tmp[rk].insert( geom ); + } + for( auto const & [rk, geom]: tmp ) { - CellGlbIdx const cgi = CellGlbIdx{ intConv< globalIndex >( index - cellOffset ) }; - ghost.cells.emplace( cgi, owner ); + GEOS_LOG_RANK( "ghost geom = " << rk << ": " << json( geom ) ); } } + GEOS_LOG_RANK( "my final neighbors are " << json( ghost.neighbors ) ); return ghost; -// if( curRank == 2_mpi ) -// { -// std::map< MpiRank, std::set< CellGlbIdx > > tmp; -// for( auto const & [geom, rk]: cells ) -// { -// tmp[rk].insert( geom ); -// } -// for( auto const & [rk, geom]: tmp ) -// { -// GEOS_LOG_RANK( "ghost geom = " << rk << ": " << json( geom ) ); -// } -// } -// GEOS_LOG_RANK( "my final neighbors are " << json( neighbors ) ); // TODO to build the edges -> nodes map containing the all the ghost (i.e. the ghosted ones as well), // Let's create a vector (or matrix?) full of ones where we have edges and multiply using the adjacency matrix. From 3a9946128eed892714052c274db200bd64c2d72a Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 09:06:27 -0700 Subject: [PATCH 064/106] Fix naming conventions. --- src/coreComponents/mesh/generators/NewGhosting.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index f31c545ba82..bf40d3b9708 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -195,9 +195,9 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); for( std::size_t i = 0; i < nodes.size(); ++i ) { - vtkIdType const lni = face->GetPointId( i ); - vtkIdType const gni = globalPtIds[lni]; - nodes[i] = NodeGlbIdx{ gni }; + vtkIdType const nli = face->GetPointId( i ); + vtkIdType const ngi = globalPtIds[nli]; + nodes[i] = NodeGlbIdx{ ngi }; } tmpFaces.emplace_back( reorderFaceNodes( nodes ) ); } @@ -826,9 +826,9 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su std::vector< NodeGlbIdx > faceNodes( pids->GetNumberOfIds() ); for( std::size_t i = 0; i < faceNodes.size(); ++i ) { - vtkIdType const lni = face->GetPointId( i ); - vtkIdType const gni = globalPtIds->GetValue( lni ); - faceNodes[i] = NodeGlbIdx{ gni }; + vtkIdType const nli = face->GetPointId( i ); + vtkIdType const ngi = globalPtIds->GetValue( nli ); + faceNodes[i] = NodeGlbIdx{ ngi }; } std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes ); result.c2f[gci].insert( n2f.at( reorderedFaceNodes ) ); From 3217ee2d5c3b9d2e19da070ffe28934417748093 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 09:34:28 -0700 Subject: [PATCH 065/106] Comments --- src/coreComponents/mesh/generators/NewGhosting.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index bf40d3b9708..9a6603d4935 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1199,6 +1199,8 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, { EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); ghost.edges.emplace( egi, owner ); + // TODO make all the following check in on time with sets comparison instead of "point-wise" comparisons. + // TODO same for the faces and cells... if( graph.e2n.find( egi ) == graph.e2n.cend() and graph.otherEdges.find( egi ) == graph.otherEdges.cend() ) { missingIndices.emplace_back( index ); @@ -1251,7 +1253,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, } missing.FillComplete( ownedMap, missingIndicesMap ); - // TODO Don't put ones in downard: reassemble with the ordering (e.g. edges point to a first and last node) + // TODO Don't put ones in downward: reassemble with the ordering (e.g. edges point to a first and last node) Epetra_CrsMatrix missingMappings( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( missing, false, *tDownward, true, missingMappings, false ); missingMappings.FillComplete( ownedMap, missingIndicesMap ); From 1dcba3d12d6d6837e7c80b97f1f9af0c0d261a3c Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 15:47:29 -0700 Subject: [PATCH 066/106] Split the MeshGraph in 2 instances dedicated to owned and present geometrical quantities. --- .../mesh/generators/NewGhosting.cpp | 146 ++++++++---------- 1 file changed, 66 insertions(+), 80 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 9a6603d4935..ac21f8d0885 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -695,17 +695,12 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, return result; } -struct MeshGraph // TODO add the local <-> global mappings here? +struct MeshGraph { std::map< CellGlbIdx, std::set< FaceGlbIdx > > c2f; // TODO What about the metadata (e.g. flip the face) std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? - std::set< NodeGlbIdx > nodes; - std::set< FaceGlbIdx > otherFaces; // Faces that are there but not owned. - std::set< EdgeGlbIdx > otherEdges; // Edges that are there but not owned. - std::set< NodeGlbIdx > otherNodes; // Nodes that are there but not owned. - // TODO add the nodes here? - // TODO add all types of connections here? How? + std::set< NodeGlbIdx > n; }; void to_json( json & j, @@ -713,55 +708,54 @@ void to_json( json & j, { j = json{ { "c2f", v.f2e }, { "f2e", v.f2e }, - { "e2n", v.e2n } }; + { "e2n", v.e2n }, + { "n", v.n } }; } - /** * @brief Builds the graph information for the owned elements only. * @param mesh * @param buckets * @param offsets * @param curRank - * @return + * @return The tuple of first the owned geometrical quantities (ie keys are owned) + * and second the geometrical quantities that are present on the rank but not owned. */ -MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a sub-mesh? - Buckets const & buckets, - BucketOffsets const & offsets, - MpiRank curRank ) +std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a sub-mesh? + Buckets const & buckets, + BucketOffsets const & offsets, + MpiRank curRank ) { - MeshGraph result; + MeshGraph owned, present; auto const isCurrentRankOwning = [&curRank]( std::set< MpiRank > const & ranks ) -> bool { return curRank == *std::min_element( std::cbegin( ranks ), std::cend( ranks ) ); }; - for( auto const & [ranks, ns]: buckets.nodes ) + for( auto const & [ranks, nodes]: buckets.nodes ) { - std::set< NodeGlbIdx > & nodes = isCurrentRankOwning( ranks ) ? result.nodes : result.otherNodes; - nodes.insert( std::cbegin( ns ), std::cend( ns ) ); + std::set< NodeGlbIdx > & n = isCurrentRankOwning( ranks ) ? owned.n : present.n; + n.insert( std::cbegin( nodes ), std::cend( nodes ) ); } - // The `e2n` is a mapping for all the geometrical entities, not only the one owned like `result.e2n`. - // TODO check that it is really useful. - std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; for( auto const & [ranks, edges]: buckets.edges ) { - bool const isOwning = isCurrentRankOwning( ranks ); - auto & m = isOwning ? result.e2n : e2n; - EdgeGlbIdx i = offsets.edges.at( ranks ); // TODO hack + auto & e2n = isCurrentRankOwning( ranks ) ? owned.e2n : present.e2n; + EdgeGlbIdx egi = offsets.edges.at( ranks ); // TODO hack for( Edge const & edge: edges ) { - m[i] = edge; - if( not isOwning ) - { - result.otherEdges.insert( i ); // TODO use the keys of e2n instead? - } - ++i; + e2n[egi] = edge; + ++egi; } } - e2n.insert( std::cbegin( result.e2n ), std::cend( result.e2n ) ); + // The `e2n` is a mapping for all the geometrical entities, not only the one owned like `result.e2n`. + // TODO check that it is really useful. + std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; + for( auto const & m: { owned.e2n, present.e2n } ) + { + e2n.insert( std::cbegin( m ), std::cend( m ) ); + } // Simple inversion std::map< std::tuple< NodeGlbIdx, NodeGlbIdx >, EdgeGlbIdx > n2e; @@ -772,43 +766,32 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su for( auto const & [ranks, faces]: buckets.faces ) { - bool const isOwning = isCurrentRankOwning( ranks ); + auto & f2e = isCurrentRankOwning( ranks ) ? owned.f2e : present.f2e; - if( isOwning ) - { - FaceGlbIdx i = offsets.faces.at( ranks ); - for( Face face: faces ) // Intentional copy for the future `emplace_back`. - { - face.emplace_back( face.front() ); // Trick to build the edges. - for( std::size_t ii = 0; ii < face.size() - 1; ++ii ) - { - NodeGlbIdx const & n0 = face[ii], & n1 = face[ii + 1]; - std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); - std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); - result.f2e[i].insert( n2e.at( p1 ) ); - bool const flipped = p0 != p1; // TODO store somewhere. - } - ++i; - } - } - else + FaceGlbIdx fgi = offsets.faces.at( ranks ); + for( Face face: faces ) // Intentional copy for the future `emplace_back`. { - FaceGlbIdx const size = FaceGlbIdx{ intConv< FaceGlbIdx::UnderlyingType >( std::size( faces ) ) }; - for( FaceGlbIdx ii = offsets.faces.at( ranks ); ii < size; ++ii ) + face.emplace_back( face.front() ); // Trick to build the edges. + for( std::size_t i = 0; i < face.size() - 1; ++i ) { - result.otherFaces.insert( ii ); // TODO insert iota + NodeGlbIdx const & n0 = face[i], & n1 = face[i + 1]; + std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); + std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); + f2e[fgi].insert( n2e.at( p1 ) ); + bool const flipped = p0 != p1; // TODO store somewhere. } + ++fgi; } } std::map< std::vector< NodeGlbIdx >, FaceGlbIdx > n2f; for( auto const & [ranks, faces]: buckets.faces ) { - FaceGlbIdx i = offsets.faces.at( ranks ); + FaceGlbIdx fgi = offsets.faces.at( ranks ); for( Face const & face: faces ) // TODO hack { - n2f[face] = i; - ++i; + n2f[face] = fgi; + ++fgi; } } @@ -831,12 +814,12 @@ MeshGraph buildMeshGraph( vtkSmartPointer< vtkDataSet > mesh, // TODO give a su faceNodes[i] = NodeGlbIdx{ ngi }; } std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes ); - result.c2f[gci].insert( n2f.at( reorderedFaceNodes ) ); + owned.c2f[gci].insert( n2f.at( reorderedFaceNodes ) ); // TODO... bool const flipped = ... compare nodes and reorderedNodes. Or ask `reorderFaceNodes` to tell } } - return result; + return { std::move( owned ), std::move( present ) }; } std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, @@ -980,7 +963,8 @@ class FindGeometricalType }; -Ghost assembleAdjacencyMatrix( MeshGraph const & graph, +Ghost assembleAdjacencyMatrix( MeshGraph const & owned, + MeshGraph const & present, MaxGlbIdcs const & gis, MpiRank curRank ) { @@ -990,15 +974,15 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, std::size_t const n = cellOffset + gis.cells.get() + 1; // Total number of entries in the graph. - std::size_t const numOwnedNodes = std::size( graph.nodes ); - std::size_t const numOwnedEdges = std::size( graph.e2n ); - std::size_t const numOwnedFaces = std::size( graph.f2e ); - std::size_t const numOwnedCells = std::size( graph.c2f ); + std::size_t const numOwnedNodes = std::size( owned.n ); + std::size_t const numOwnedEdges = std::size( owned.e2n ); + std::size_t const numOwnedFaces = std::size( owned.f2e ); + std::size_t const numOwnedCells = std::size( owned.c2f ); std::size_t const numOwned = numOwnedNodes + numOwnedEdges + numOwnedFaces + numOwnedCells; - std::size_t const numOtherNodes = std::size( graph.otherNodes ); - std::size_t const numOtherEdges = std::size( graph.otherEdges ); - std::size_t const numOtherFaces = std::size( graph.otherFaces ); + std::size_t const numOtherNodes = std::size( present.n ); + std::size_t const numOtherEdges = std::size( present.e2n ); + std::size_t const numOtherFaces = std::size( present.f2e ); std::size_t const numOther = numOtherNodes + numOtherEdges + numOtherFaces; std::vector< int > ownedGlbIdcs, numEntriesPerRow; // TODO I couldn't use a vector of `std::size_t` @@ -1009,21 +993,21 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, std::vector< int > otherGlbIdcs; // TODO I couldn't use a vector of `std::size_t` otherGlbIdcs.reserve( numOther ); - for( NodeGlbIdx const & ngi: graph.otherNodes ) + for( NodeGlbIdx const & ngi: present.n ) { otherGlbIdcs.emplace_back( ngi.get() ); } - for( EdgeGlbIdx const & egi: graph.otherEdges ) + for( auto const & [egi, _]: present.e2n ) { otherGlbIdcs.emplace_back( egi.get() + edgeOffset ); } - for( FaceGlbIdx const & fgi: graph.otherFaces ) + for( auto const & [fgi, _]: present.f2e ) { otherGlbIdcs.emplace_back( fgi.get() + faceOffset ); } GEOS_ASSERT_EQ( numOther, std::size( otherGlbIdcs ) ); - for( NodeGlbIdx const & ngi: graph.nodes ) + for( NodeGlbIdx const & ngi: owned.n ) { auto const i = ngi.get(); ownedGlbIdcs.emplace_back( i ); @@ -1031,7 +1015,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, std::vector< int > const tmp( 1, ownedGlbIdcs.back() ); indices.emplace_back( tmp ); } - for( auto const & [egi, nodes]: graph.e2n ) + for( auto const & [egi, nodes]: owned.e2n ) { auto const i = egi.get() + edgeOffset; ownedGlbIdcs.emplace_back( i ); @@ -1039,7 +1023,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, std::vector< int > const tmp{ int( std::get< 0 >( nodes ).get() ), int( std::get< 1 >( nodes ).get() ), ownedGlbIdcs.back() }; indices.emplace_back( tmp ); } - for( auto const & [fgi, edges]: graph.f2e ) + for( auto const & [fgi, edges]: owned.f2e ) { auto const i = fgi.get() + faceOffset; ownedGlbIdcs.emplace_back( i ); @@ -1053,7 +1037,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, tmp.emplace_back( ownedGlbIdcs.back() ); indices.emplace_back( tmp ); } - for( auto const & [cgi, faces]: graph.c2f ) + for( auto const & [cgi, faces]: owned.c2f ) { auto const i = cgi.get() + cellOffset; ownedGlbIdcs.emplace_back( i ); @@ -1201,7 +1185,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, ghost.edges.emplace( egi, owner ); // TODO make all the following check in on time with sets comparison instead of "point-wise" comparisons. // TODO same for the faces and cells... - if( graph.e2n.find( egi ) == graph.e2n.cend() and graph.otherEdges.find( egi ) == graph.otherEdges.cend() ) + if( owned.e2n.find( egi ) == owned.e2n.cend() and present.e2n.find( egi ) == present.e2n.cend() ) { missingIndices.emplace_back( index ); } @@ -1211,7 +1195,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, { FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); ghost.faces.emplace( fgi, owner ); - if( graph.f2e.find( fgi ) == graph.f2e.cend() and graph.otherFaces.find( fgi ) == graph.otherFaces.cend() ) + if( owned.f2e.find( fgi ) == owned.f2e.cend() and present.f2e.find( fgi ) == present.f2e.cend() ) { missingIndices.emplace_back( index ); } @@ -1221,7 +1205,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & graph, { CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); ghost.cells.emplace( cgi, owner ); - if( graph.c2f.find( cgi ) == graph.c2f.cend() ) + if( owned.c2f.find( cgi ) == owned.c2f.cend() ) { missingIndices.emplace_back( index ); } @@ -1444,13 +1428,15 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ) - 1_egi, offsets.faces.at( { nextRank } ) - 1_fgi ); std::cout << "matrixOffsets on rank " << curRank << " -> " << json( matrixOffsets ) << std::endl; - MeshGraph const graph = buildMeshGraph( mesh, buckets, offsets, curRank ); // TODO change into buildOwnedMeshGraph? -// if( curRank == MpiRank{ 1 } ) + auto const [owned, present] = buildMeshGraph( mesh, buckets, offsets, curRank ); // TODO change into buildOwnedMeshGraph? +// if( curRank == 1_mpi ) // { -// std::cout << "My graph is " << json( graph ) << std::endl; +// GEOS_LOG_RANK( "My owned is " << json( owned ) ); +// GEOS_LOG_RANK( "My present is " << json( present ) ); // } +// MpiWrapper::barrier(); - Ghost const ghost = assembleAdjacencyMatrix( graph, matrixOffsets, curRank ); + Ghost const ghost = assembleAdjacencyMatrix( owned, present, matrixOffsets, curRank ); buildPods( ghost ); From b04001247281b33ce3b6d98950364e8fef8a309d Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 16:20:37 -0700 Subject: [PATCH 067/106] Create and return a Ghost graph + rename `Ghost` struct to `Ownerships` --- .../mesh/generators/NewGhosting.cpp | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index ac21f8d0885..0bbb967be2c 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -837,7 +837,7 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, /** * Contains the full result of the ghosting */ -struct Ghost +struct Ownerships { std::map< NodeGlbIdx, MpiRank > nodes; std::map< EdgeGlbIdx, MpiRank > edges; @@ -963,10 +963,10 @@ class FindGeometricalType }; -Ghost assembleAdjacencyMatrix( MeshGraph const & owned, - MeshGraph const & present, - MaxGlbIdcs const & gis, - MpiRank curRank ) +std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & owned, + MeshGraph const & present, + MaxGlbIdcs const & gis, + MpiRank curRank ) { std::size_t const edgeOffset = gis.nodes.get() + 1; std::size_t const faceOffset = edgeOffset + gis.edges.get() + 1; @@ -1160,7 +1160,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, FindGeometricalType const getGeomType( edgeOffset, faceOffset, cellOffset ); using Geom = FindGeometricalType::Geom; - Ghost ghost; + Ownerships ownerships; for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? { int const & index = extractedIndices[i]; @@ -1168,7 +1168,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; if( owner != curRank ) { - ghost.neighbors.insert( owner ); + ownerships.neighbors.insert( owner ); } switch( getGeomType( index ) ) @@ -1176,13 +1176,13 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, case Geom::NODE: { NodeGlbIdx const ngi = getGeomType.as< NodeGlbIdx >( index ); - ghost.nodes.emplace( ngi, owner ); + ownerships.nodes.emplace( ngi, owner ); break; } case Geom::EDGE: { EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); - ghost.edges.emplace( egi, owner ); + ownerships.edges.emplace( egi, owner ); // TODO make all the following check in on time with sets comparison instead of "point-wise" comparisons. // TODO same for the faces and cells... if( owned.e2n.find( egi ) == owned.e2n.cend() and present.e2n.find( egi ) == present.e2n.cend() ) @@ -1194,7 +1194,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, case Geom::FACE: { FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); - ghost.faces.emplace( fgi, owner ); + ownerships.faces.emplace( fgi, owner ); if( owned.f2e.find( fgi ) == owned.f2e.cend() and present.f2e.find( fgi ) == present.f2e.cend() ) { missingIndices.emplace_back( index ); @@ -1204,7 +1204,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, case Geom::CELL: { CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); - ghost.cells.emplace( cgi, owner ); + ownerships.cells.emplace( cgi, owner ); if( owned.c2f.find( cgi ) == owned.c2f.cend() ) { missingIndices.emplace_back( index ); @@ -1243,7 +1243,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, missingMappings.FillComplete( ownedMap, missingIndicesMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingMappings ); - MeshGraph neighboringConnexions; + MeshGraph ghosts; int ext = 0; std::vector< double > extValues( n ); @@ -1270,14 +1270,14 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); NodeGlbIdx const ngi0 = getGeomType.as< NodeGlbIdx >( extIndices[0] ); NodeGlbIdx const ngi1 = getGeomType.as< NodeGlbIdx >( extIndices[1] ); - neighboringConnexions.e2n[egi] = std::minmax( { ngi0, ngi1 } ); + ghosts.e2n[egi] = std::minmax( { ngi0, ngi1 } ); break; } case Geom::FACE: { GEOS_ASSERT_EQ( ext, 4 ); // TODO temporary check for the dev FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); - std::set< EdgeGlbIdx > & tmp = neighboringConnexions.f2e[fgi]; + std::set< EdgeGlbIdx > & tmp = ghosts.f2e[fgi]; for( int ii = 0; ii < ext; ++ii ) { tmp.insert( EdgeGlbIdx{ intConv< globalIndex >( extIndices[ii] - edgeOffset ) } ); @@ -1288,7 +1288,7 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, { GEOS_ASSERT_EQ( ext, 6 ); // TODO temporary check for the dev CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); - std::set< FaceGlbIdx > & tmp = neighboringConnexions.c2f[cgi]; + std::set< FaceGlbIdx > & tmp = ghosts.c2f[cgi]; for( int ii = 0; ii < ext; ++ii ) { tmp.insert( FaceGlbIdx{ intConv< globalIndex >( extIndices[ii] - faceOffset ) } ); @@ -1304,9 +1304,9 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, if( curRank == 2_mpi ) { - GEOS_LOG_RANK( "neighboringConnexions = " << json( neighboringConnexions ) ); + GEOS_LOG_RANK( "neighboringConnexions = " << json( ghosts ) ); std::map< MpiRank, std::set< CellGlbIdx > > tmp; - for( auto const & [geom, rk]: ghost.cells ) + for( auto const & [geom, rk]: ownerships.cells ) { tmp[rk].insert( geom ); } @@ -1315,9 +1315,9 @@ Ghost assembleAdjacencyMatrix( MeshGraph const & owned, GEOS_LOG_RANK( "ghost geom = " << rk << ": " << json( geom ) ); } } - GEOS_LOG_RANK( "my final neighbors are " << json( ghost.neighbors ) ); + GEOS_LOG_RANK( "my final neighbors are " << json( ownerships.neighbors ) ); - return ghost; + return { std::move( ghosts ), std::move( ownerships ) }; // TODO to build the edges -> nodes map containing the all the ghost (i.e. the ghosted ones as well), @@ -1349,13 +1349,16 @@ buildGhostRankAndL2G( std::map< GI, MpiRank > const & m ) return std::make_tuple( ghostRank, l2g ); } -void buildPods( Ghost const & ghost ) +void buildPods( MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts, + Ownerships const & ownerships ) { - std::size_t const numNodes = std::size( ghost.nodes ); - std::size_t const numEdges = std::size( ghost.edges ); - std::size_t const numFaces = std::size( ghost.faces ); + std::size_t const numNodes = std::size( ownerships.nodes ); + std::size_t const numEdges = std::size( ownerships.edges ); + std::size_t const numFaces = std::size( ownerships.faces ); - auto [ghostRank, l2g] = buildGhostRankAndL2G( ghost.edges ); + auto [ghostRank, l2g] = buildGhostRankAndL2G( ownerships.edges ); NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); @@ -1436,9 +1439,9 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v // } // MpiWrapper::barrier(); - Ghost const ghost = assembleAdjacencyMatrix( owned, present, matrixOffsets, curRank ); + auto const [ghosts, ownerships] = assembleAdjacencyMatrix( owned, present, matrixOffsets, curRank ); - buildPods( ghost ); + buildPods( owned, present, ghosts, ownerships ); return {}; } From 385986cd6b484d5e58f980e0170cef59af1652d5 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 16:24:02 -0700 Subject: [PATCH 068/106] remove wrong doxygen --- src/coreComponents/mesh/generators/NewGhosting.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 0bbb967be2c..291a1187b8b 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -834,9 +834,7 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, return ptr; } -/** - * Contains the full result of the ghosting - */ + struct Ownerships { std::map< NodeGlbIdx, MpiRank > nodes; From 4b2e61c1995e675d8dccdacd1344d500c63d5654 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 16:43:55 -0700 Subject: [PATCH 069/106] Added a dedicated file for the global numbering. --- src/coreComponents/mesh/CMakeLists.txt | 2 ++ .../mesh/generators/NewGhosting.cpp | 3 +++ .../mesh/generators/NewGlobalNumbering.cpp | 25 +++++++++++++++++++ .../mesh/generators/NewGlobalNumbering.hpp | 25 +++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 src/coreComponents/mesh/generators/NewGlobalNumbering.cpp create mode 100644 src/coreComponents/mesh/generators/NewGlobalNumbering.hpp diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index be5ea58c8d9..2fa54d84087 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -184,6 +184,7 @@ if( ENABLE_VTK ) set( mesh_headers ${mesh_headers} generators/CollocatedNodes.hpp generators/NewGhosting.hpp + generators/NewGlobalNumbering.hpp generators/VTKFaceBlockUtilities.hpp generators/VTKMeshGenerator.hpp generators/VTKMeshGeneratorTools.hpp @@ -193,6 +194,7 @@ if( ENABLE_VTK ) set( mesh_sources ${mesh_sources} generators/CollocatedNodes.cpp generators/NewGhosting.cpp + generators/NewGlobalNumbering.cpp generators/VTKFaceBlockUtilities.cpp generators/VTKMeshGenerator.cpp generators/VTKMeshGeneratorTools.cpp diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 291a1187b8b..06e4b992e1a 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -14,6 +14,8 @@ #include "NewGhosting.hpp" +#include "NewGlobalNumbering.hpp" + #include "mesh/mpiCommunications/CommunicationTools.hpp" #include "mesh/mpiCommunications/MPI_iCommData.hpp" #include "mesh/mpiCommunications/NeighborCommunicator.hpp" @@ -1366,6 +1368,7 @@ void buildPods( MeshGraph const & owned, std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { + doTheNewGlobalNumbering(); // Now we exchange the data with our neighbors. MpiRank const curRank{ MpiWrapper::commRank() }; diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp new file mode 100644 index 00000000000..60f96359a3e --- /dev/null +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -0,0 +1,25 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "NewGlobalNumbering.hpp" + +namespace geos::ghosting +{ + +void doTheNewGlobalNumbering() +{ + +} + +} // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp new file mode 100644 index 00000000000..fd1db1078a9 --- /dev/null +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp @@ -0,0 +1,25 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_NEWGLOBALNUMBERING_HPP +#define GEOS_NEWGLOBALNUMBERING_HPP + +namespace geos::ghosting +{ + +void doTheNewGlobalNumbering(); + +} // end of namespace + +#endif //GEOS_NEWGLOBALNUMBERING_HPP From ff8e76575f1613ae53c60b824a787d36e64d06f1 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 17:19:29 -0700 Subject: [PATCH 070/106] Split global numbering and ghosting --- .../mesh/generators/NewGhosting.cpp | 644 +----------------- .../mesh/generators/NewGlobalNumbering.cpp | 576 +++++++++++++++- .../mesh/generators/NewGlobalNumbering.hpp | 67 +- 3 files changed, 644 insertions(+), 643 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 06e4b992e1a..fd6f89947fc 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -16,10 +16,6 @@ #include "NewGlobalNumbering.hpp" -#include "mesh/mpiCommunications/CommunicationTools.hpp" -#include "mesh/mpiCommunications/MPI_iCommData.hpp" -#include "mesh/mpiCommunications/NeighborCommunicator.hpp" - #include "Pods.hpp" #include "common/MpiWrapper.hpp" @@ -38,8 +34,6 @@ #include #include -#include -#include #include @@ -51,582 +45,6 @@ using json = nlohmann::json; namespace geos::ghosting { -using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; -using Face = std::vector< NodeGlbIdx >; - -struct Exchange -{ - std::set< NodeGlbIdx > nodes; - std::set< Edge > edges; - std::set< Face > faces; -}; - -void to_json( json & j, - const Exchange & v ) -{ - j = json{ { "nodes", v.nodes }, - { "edges", v.edges }, - { "faces", v.faces } }; -} - -void from_json( const json & j, - Exchange & v ) -{ - v.nodes = j.at( "nodes" ).get< std::set< NodeGlbIdx > >(); - v.edges = j.at( "edges" ).get< std::set< Edge > >(); - v.faces = j.at( "faces" ).get< std::set< Face > >(); -} - -/** - * @brief Extract the cells at the boundary of the mesh. - * @param mesh The vtk mesh. - * @return The vtk cell ids. - */ -std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) -{ - // TODO Better handle the boundary information, forgetting about the 3d cells and simply handling the outside shell. - auto f = vtkDataSetSurfaceFilter::New(); - f->PassThroughCellIdsOn(); - f->PassThroughPointIdsOff(); - f->FastModeOff(); - - string const originalCellsKey = "ORIGINAL_CELLS"; - f->SetOriginalCellIdsName( originalCellsKey.c_str() ); - auto boundaryMesh = vtkPolyData::New(); - f->UnstructuredGridExecute( mesh, boundaryMesh ); - vtkIdTypeArray const * originalCells = vtkIdTypeArray::FastDownCast( boundaryMesh->GetCellData()->GetArray( originalCellsKey.c_str() ) ); - - std::set< vtkIdType > boundaryCellIdxs; - for( auto i = 0; i < originalCells->GetNumberOfTuples(); ++i ) - { - boundaryCellIdxs.insert( originalCells->GetValue( i ) ); - } - - return boundaryCellIdxs; -} - -/** - * @brief Order the nodes of the faces in a way that can be reproduced across the MPI ranks. - * @param nodes The list of nodes as provided by the mesh. - * @return A face with the nodes in the appropriate order - * @details The nodes will be ordered in the following way. - * First, we look for the lowest node index. It will become the first node. - * Then we must pick the second node. We have to choices: just before or just after the first. - * (we do not want to shuffle the nodes completely, we need to keep track of the order). - * To do this, we select the nodes with the lowest index as well. - * This also defines a direction in which we'll pick the other nodes. - * For example, the face [2, 3, 1, 5, 9, 8] will become [1, 3, 2, 8, 9, 5] - * because we'll start with @c 1 and then select the @c 3 over the @c 5. - * Which defines the direction @c 2, @c 8, @c 9, @c 5. - * @note This is the same pattern that we apply for edges. - * Except that edges having only two nodes, it's not necessary to implement a dedicated function - * and std::minmax is enough. - */ -Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) -{ - std::size_t const n = nodes.size(); - - // Handles negative values of `i`. - auto const modulo = [n]( integer const & i ) -> std::size_t - { - integer mod = i % n; - if( mod < 0 ) - { - mod += n; - } - return mod; - }; - - Face f; - f.reserve( n ); - - auto const it = std::min_element( nodes.cbegin(), nodes.cend() ); - std::size_t const minIdx = std::distance( nodes.cbegin(), it ); - int const increment = nodes[modulo( minIdx - 1 )] < nodes[modulo( minIdx + 1 )] ? -1 : 1; // TODO based on increment, I can say if the face is flipped or not. - integer i = minIdx; - for( std::size_t count = 0; count < n; ++count, i = i + increment ) - { - f.emplace_back( nodes.at( modulo( i ) ) ); - } - - return f; -} - -Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, - std::set< vtkIdType > const & cellIds ) -{ - vtkIdTypeArray * gids = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); - Span< vtkIdType > const globalPtIds( (vtkIdType *) gids->GetPointer( 0 ), gids->GetNumberOfTuples() ); - - std::vector< Edge > tmpEdges; - std::vector< Face > tmpFaces; - - // Pre-allocation of the temporary vectors. - { - std::size_t numEdges = 0; - std::size_t numFaces = 0; - for( vtkIdType const & c: cellIds ) - { - vtkCell * cell = mesh->GetCell( c ); - numEdges += cell->GetNumberOfEdges(); - numFaces += cell->GetNumberOfFaces(); - } - tmpEdges.reserve( numEdges ); - tmpFaces.reserve( numFaces ); - } - - // Filling the temporary. - for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) - { - vtkCell * cell = mesh->GetCell( c ); - - for( auto e = 0; e < cell->GetNumberOfEdges(); ++e ) - { - vtkCell * edge = cell->GetEdge( e ); - vtkIdType const ln0 = edge->GetPointId( 0 ); - vtkIdType const ln1 = edge->GetPointId( 1 ); - vtkIdType const gn0 = globalPtIds[ln0]; - vtkIdType const gn1 = globalPtIds[ln1]; - tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ gn0 }, NodeGlbIdx{ gn1 } } ) ); - } - - for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) - { - vtkCell * face = cell->GetFace( f ); - vtkIdList * pids = face->GetPointIds(); - std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); - for( std::size_t i = 0; i < nodes.size(); ++i ) - { - vtkIdType const nli = face->GetPointId( i ); - vtkIdType const ngi = globalPtIds[nli]; - nodes[i] = NodeGlbIdx{ ngi }; - } - tmpFaces.emplace_back( reorderFaceNodes( nodes ) ); - } - } - - std::set< NodeGlbIdx > nodes; - std::transform( std::cbegin( globalPtIds ), std::cend( globalPtIds ), - std::inserter( nodes, std::end( nodes ) ), []( vtkIdType const & id ) - { return NodeGlbIdx{ id }; } ); - - // Removing the duplicates by copying into a `std::set`. - std::set< Edge > edges{ tmpEdges.cbegin(), tmpEdges.cend() }; // SortedArray requires the input to be sorted already. - std::set< Face > faces{ tmpFaces.cbegin(), tmpFaces.cend() }; - - return { std::move( nodes ), std::move( edges ), std::move( faces ) }; -} - -array1d< std::uint8_t > convertExchange( Exchange const & exchange ) -{ - std::vector< std::uint8_t > const tmp = json::to_cbor( json( exchange ) ); - array1d< std::uint8_t > result; - result.reserve( tmp.size() ); - for( std::uint8_t const & t: tmp ) - { - result.emplace_back( t ); - } - return result; -} - -Exchange convertExchange( array1d< std::uint8_t > const & input ) -{ - std::vector< std::uint8_t > const tmp( std::cbegin( input ), std::cend( input ) ); - return json::from_cbor( tmp, false ).template get< Exchange >(); -} - -Exchange buildFullData( vtkSmartPointer< vtkDataSet > mesh ) -{ - std::set< vtkIdType > cellIds; - for( vtkIdType i = 0; i < mesh->GetNumberOfCells(); ++i ) - { - cellIds.insert( cellIds.end(), i ); - } - - return buildSpecificData( mesh, cellIds ); -} - - -Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) -{ - return buildSpecificData( mesh, extractBoundaryCells( mesh ) ); -} - -struct Buckets -{ - std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > nodes; - std::map< std::set< MpiRank >, std::set< Edge > > edges; - std::map< std::set< MpiRank >, std::set< Face > > faces; -}; - -//void to_json( json & j, -// const Buckets & v ) -//{ -// j = json{ { "nodes", v.nodes }, { "edges", v.edges }, { "faces", v.faces } }; -//} - -/** - * @brief - * @tparam T Typically NodeGloIdx or a container of NodeGlbIdx (for edges and faces) - * @param exchanged - * @param curRank - * @param neighbors - * @return - */ -template< typename T > -std::map< std::set< MpiRank >, std::set< T > > -buildIntersectionBuckets( std::map< MpiRank, std::set< T > const & > const & exchanged, - MpiRank curRank, - std::set< MpiRank > const & neighbors ) -{ - std::map< T, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? - // We "register" all the edges of the current rank: they are the only one we're interested in. - for( T const & node: exchanged.at( curRank ) ) - { - counts.emplace_hint( counts.end(), node, std::set< MpiRank >{ curRank } ); - } - - // We now loop on the neighbor edges. - // If a neighbor has an edge in common with the current rank, they we store it. - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. - { - for( T const & node: exchanged.at( neighborRank ) ) - { - auto it = counts.find( node ); - if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. - { - it->second.insert( neighborRank ); - } - } - } - - std::map< std::set< MpiRank >, std::set< T > > nodeBuckets; - for( auto const & [node, ranks]: counts ) - { - if( ranks.find( curRank ) != ranks.cend() ) - { - nodeBuckets[ranks].insert( node ); - } - } - return nodeBuckets; -} - - -/** - * @brief Compute the intersection between for the ranks based on the information @p exchanged. - * @param exchanged Geometrical information provided by the ranks (including the current rank). - * @param curRank The current MPI rank. - * @param neighbors excluding current rank - * @return The intersection buckets. - */ -Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, - MpiRank curRank, - std::set< MpiRank > const & neighbors ) -{ - std::map< MpiRank, std::set< NodeGlbIdx > const & > nodeInfo; - for( auto const & [rank, exchange]: exchanged ) - { - nodeInfo.emplace( rank, exchange.nodes ); - } - std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > const nodeBuckets = buildIntersectionBuckets( nodeInfo, curRank, neighbors ); - - std::map< MpiRank, std::set< Edge > const & > edgeInfo; - for( auto const & [rank, exchange]: exchanged ) - { - edgeInfo.emplace( rank, exchange.edges ); - } - std::map< std::set< MpiRank >, std::set< Edge > > const edgeBuckets = buildIntersectionBuckets( edgeInfo, curRank, neighbors ); - - // For faces, the algorithm can be a tad simpler because faces can be shared by at most 2 ranks. - std::map< std::set< MpiRank >, std::set< Face > > faceBuckets; - std::set< Face > curFaces = exchanged.at( curRank ).faces; - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. - { - for( Face const & face: exchanged.at( neighborRank ).faces ) - { - auto it = curFaces.find( face ); - if( it != curFaces.cend() ) - { - faceBuckets[{ curRank, neighborRank }].insert( *it ); - curFaces.erase( it ); - } - } - } - faceBuckets[{ curRank }] = curFaces; - -// // Checking if neighbors is too wide... // TODO do we care here? -// std::set< MpiRank > usefulNeighbors; -// for( auto const & [ranks, nodes]: nodeBuckets ) -// { -// if( not nodes.empty() ) -// { -// usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); -// } -// } -// for( auto const & [ranks, edges]: edgeBuckets ) -// { -// if( not edges.empty() ) -// { -// usefulNeighbors.insert( ranks.cbegin(), ranks.cend() ); -// } -// } -// std::vector< MpiRank > uselessNeighbors; -// std::set_difference( neighbors.cbegin(), neighbors.cend(), usefulNeighbors.cbegin(), usefulNeighbors.cend(), std::back_inserter( uselessNeighbors ) ); -// // TODO... Remove the neighbors? - - return { nodeBuckets, edgeBuckets, faceBuckets }; -} - - -// TODO Duplicated -std::map< MpiRank, Exchange > exchange( int commId, - std::vector< NeighborCommunicator > & neighbors, - Exchange const & data ) -{ - MPI_iCommData commData( commId ); - integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); - commData.resize( numNeighbors ); - - - array1d< std::uint8_t > const cv = convertExchange( data ); - - for( integer i = 0; i < numNeighbors; ++i ) - { - neighbors[i].mpiISendReceiveSizes( cv, - commData.mpiSendBufferSizeRequest( i ), - commData.mpiRecvBufferSizeRequest( i ), - commId, - MPI_COMM_GEOSX ); - } - - MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus() ); - MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus() ); - - array1d< array1d< std::uint8_t > > rawExchanged( neighbors.size() ); - - for( integer i = 0; i < numNeighbors; ++i ) - { - neighbors[i].mpiISendReceiveData( cv, - commData.mpiSendBufferRequest( i ), - rawExchanged[i], - commData.mpiRecvBufferRequest( i ), - commId, - MPI_COMM_GEOSX ); - } - MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferRequest(), commData.mpiSendBufferStatus() ); - MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferRequest(), commData.mpiRecvBufferStatus() ); - - std::map< MpiRank, Exchange > output; - for( auto i = 0; i < numNeighbors; ++i ) - { - output[MpiRank{ neighbors[i].neighborRank() }] = convertExchange( rawExchanged[i] ); - } - return output; -} - - -/** - * @brief Estimate an upper bound of the serialized size of the size @p buckets. - * @param buckets - * @return Size in bytes. - * @details @c MPI_Scan requires a fixed buffer size. - * An a-priori estimation of the maximum size of the buffer leads to crazy amounts of memory - * because of the combinatorial pattern of the rank intersections. - * Therefore, we use an initial MPI communication to get a better estimation. - * @node We don't care about the @c nodes because their global numbering is already done. - */ -std::size_t buildMaxBufferSize( Buckets const & buckets ) // TODO give the size buckets instead? -{ - auto const f = []( auto const & bucket ) -> std::size_t - { - std::size_t size = std::size( bucket ); - for( auto const & [ranks, _]: bucket ) - { - size += std::size( ranks ) + 1 + 1; // One `+1` for the size of the ranks set, the other one for the offset - } - return size; - }; - - return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); // Add the ScannedOffsets::{edge, face}Restart -} - -struct BucketSizes -{ - using mapping = std::map< std::set< MpiRank >, localIndex >; - mapping edges; - mapping faces; -}; - -void to_json( json & j, - const BucketSizes & v ) -{ - j = json{ { "edges", v.edges }, - { "faces", v.faces } }; -} - -void from_json( const json & j, - BucketSizes & v ) -{ - v.edges = j.at( "edges" ).get< BucketSizes::mapping >(); - v.faces = j.at( "faces" ).get< BucketSizes::mapping >(); -} - -struct BucketOffsets -{ - std::map< std::set< MpiRank >, EdgeGlbIdx > edges; - std::map< std::set< MpiRank >, FaceGlbIdx > faces; -}; - -void to_json( json & j, - const BucketOffsets & v ) -{ - j = json{ { "edges", v.edges }, - { "faces", v.faces } }; -} - -void from_json( const json & j, - BucketOffsets & v ) -{ - v.edges = j.at( "edges" ).get< std::map< std::set< MpiRank >, EdgeGlbIdx > >(); - v.faces = j.at( "faces" ).get< std::map< std::set< MpiRank >, FaceGlbIdx > >(); -} - -/** - * @brief Compute the sizes of the intersection buckets. - * @param buckets The intersection buckets. - * @return The buckets of sizes. - */ -BucketSizes getBucketSize( Buckets const & buckets ) -{ - BucketSizes output; - - for( auto const & [ranks, edges]: buckets.edges ) - { - output.edges.emplace_hint( output.edges.end(), ranks, std::size( edges ) ); - } - - for( auto const & [ranks, faces]: buckets.faces ) - { - output.faces.emplace_hint( output.faces.end(), ranks, std::size( faces ) ); - } - - return output; -} - -/** - * @brief - * @tparam GLB_IDX The global index type of the geometrical quantity considered (typically @c EdgeGlbIdx or @c FaceGlbIdx). - * @param sizes The sizes of the intersection bucket (from the current MPI rank). - * @param offsets The offsets for each intersection bucket (from the MPI_Scan process, i.e. the previous ranks). - * @param curRank Current MPI rank. - * @return The offset buckets, updated with the sizes of the current MPI rank. - */ -template< typename GLB_IDX > -std::map< std::set< MpiRank >, GLB_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, - std::map< std::set< MpiRank >, GLB_IDX > const & offsets, - MpiRank curRank ) -{ - std::map< std::set< MpiRank >, GLB_IDX > reducedOffsets; - - // We only keep the offsets that are still relevant to the current and higher ranks. - // Note that `reducedOffsets` will be used by the _current_ rank too. - // Therefore, we'll send information that may not be useful to higher ranks. - // This is surely acceptable because the information is tiny. - for( auto const & [ranks, offset]: offsets ) - { - MpiRank const maxConcerned = *std::max_element( ranks.cbegin(), ranks.cend() ); - // TODO invert -// if( maxConcerned >= curRank ) -// { -// reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); -// } - if( maxConcerned < curRank ) - { - continue; - } - reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); - } - - // Add the offsets associated to the new size buckets from the current rank. - GLB_IDX nextOffset{ 0 }; - for( auto const & [ranks, size]: sizes ) - { - auto const it = reducedOffsets.find( ranks ); - if( it == reducedOffsets.end() ) - { - reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, nextOffset ); - nextOffset += GLB_IDX{ size }; - } - else - { - nextOffset = it->second + GLB_IDX{ size }; // Define the new offset from the last - } - } - - // Add an extra entry based for the following rank - reducedOffsets.emplace_hint( reducedOffsets.end(), std::set< MpiRank >{ curRank + 1_mpi }, nextOffset ); - - return reducedOffsets; -} - -std::vector< std::uint8_t > serialize( BucketSizes const & sizes ) -{ - return json::to_cbor( json( sizes ) ); -} - -std::vector< std::uint8_t > serialize( BucketOffsets const & offsets ) -{ - return json::to_cbor( json( offsets ) ); -} - -/** - * @brief - * @tparam V Container of std::uint8_t - * @param data - * @return - */ -template< class V > -BucketOffsets deserialize( V const & data ) -{ - return json::from_cbor( data, false ).template get< BucketOffsets >(); -} - -/** - * @brief Custom MPI reduction function. Merges the sizes of the bucket from current rank into numbering offsets. - * @param[in] in Contains the reduced result from the previous ranks. Needs to be unpacked into a @c BucketOffsets instance. - * @param[inout] inout As an @e input, contains a pointer to the @c BucketSizes instance from the current rank. - * As an @e output, will contained the (packed) instance of @c BucketOffsets after the @c reduction. - * The @e output instance will be used by both current and next ranks. - * @param[in] len Number of data. - * @param[in] dataType Type of data. Mean to be @c MPI_BYTE for the current case. - */ -void f( void * in, - void * inout, - int * len, - MPI_Datatype * dataType ) -{ - GEOS_ASSERT_EQ( *dataType, MPI_BYTE ); - - MpiRank const curRank{ MpiWrapper::commRank() }; - - // offsets provided by the previous rank(s) - BucketOffsets const offsets = deserialize( Span< std::uint8_t >( (std::uint8_t *) in, *len ) ); - - // Sizes provided by the current rank, under the form of a pointer to the data. - // No need to serialize, since we're on the same rank. - std::uintptr_t addr; - std::memcpy( &addr, inout, sizeof( std::uintptr_t ) ); - BucketSizes const * sizes = reinterpret_cast(addr); - - BucketOffsets updatedOffsets; - updatedOffsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes->edges, offsets.edges, curRank ); - updatedOffsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes->faces, offsets.faces, curRank ); - - // Serialize the updated offsets, so they get sent to the next rank. - std::vector< std::uint8_t > const serialized = serialize( updatedOffsets ); - std::memcpy( inout, serialized.data(), serialized.size() ); -} - struct MaxGlbIdcs { NodeGlbIdx nodes; @@ -836,7 +254,6 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, return ptr; } - struct Ownerships { std::map< NodeGlbIdx, MpiRank > nodes; @@ -846,7 +263,6 @@ struct Ownerships std::set< MpiRank > neighbors; }; - Epetra_CrsMatrix multiply( int commSize, Epetra_CrsMatrix const & indicator, Epetra_CrsMatrix & upward ) @@ -962,7 +378,6 @@ class FindGeometricalType int const m_cellOffset; }; - std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & owned, MeshGraph const & present, MaxGlbIdcs const & gis, @@ -1368,65 +783,12 @@ void buildPods( MeshGraph const & owned, std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { - doTheNewGlobalNumbering(); + auto const [buckets, offsets] = doTheNewGlobalNumbering( mesh, neighbors ); + // Now we exchange the data with our neighbors. MpiRank const curRank{ MpiWrapper::commRank() }; - std::vector< NeighborCommunicator > ncs; - ncs.reserve( neighbors.size() ); - for( MpiRank const & rank: neighbors ) - { - ncs.emplace_back( rank.get() ); - } - - CommID const commId = CommunicationTools::getInstance().getCommID(); - std::map< MpiRank, Exchange > exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); - exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); - -// std::cout << "exchanged on rank " << curRank << " -> " << json( exchanged[MpiRank{ MpiWrapper::commRank() }] ) << std::endl; - - Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); - - std::size_t const maxBufferSize = 100 * buildMaxBufferSize( buckets ); - - std::vector< std::uint8_t > sendBuffer( maxBufferSize ); - std::vector< std::uint8_t > recvBuffer( maxBufferSize ); - - BucketSizes const sizes = getBucketSize( buckets ); - BucketOffsets offsets; - if( curRank == 0_mpi ) - { - // The `MPI_Scan` process will not call the reduction operator for rank 0. - // So we need to reduce ourselves for ourselves. - offsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes.edges, { { { 0_mpi, }, 0_egi } }, curRank ); - offsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes.faces, { { { 0_mpi, }, 0_fgi } }, curRank ); - // Still we need to send this reduction to the following rank, by copying to it to the send buffer. - std::vector< std::uint8_t > const bytes = serialize( offsets ); - std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); - } - else - { - // For the other ranks, the reduction operator will be called during the `Mpi_Scan` process. - // So unlike for rank 0, we do not have to do it ourselves. - // In order to provide the `sizes` to the reduction operator, - // since `sizes` will only be used on the current rank, - // we'll provide the information as a pointer to the instance. - // The reduction operator will then compute the new offsets and send them to the following rank. - std::uintptr_t const addr = reinterpret_cast(&sizes); - std::memcpy( sendBuffer.data(), &addr, sizeof( std::uintptr_t ) ); - } - - MPI_Op op; - MPI_Op_create( f, false, &op ); - - MPI_Scan( sendBuffer.data(), recvBuffer.data(), maxBufferSize, MPI_BYTE, op, MPI_COMM_WORLD ); - - if( curRank != 0_mpi ) - { - offsets = deserialize( recvBuffer ); - } - - std::cout << "offsets on rank " << curRank << " -> " << json( offsets ) << std::endl; + GEOS_LOG_RANK( "offsets on rank " << curRank << " -> " << json( offsets ) ); MpiRank const nextRank = curRank + 1_mpi; MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ) - 1_egi, offsets.faces.at( { nextRank } ) - 1_fgi ); diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp index 60f96359a3e..d82e8890be5 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -14,12 +14,586 @@ #include "NewGlobalNumbering.hpp" +#include "mesh/mpiCommunications/CommunicationTools.hpp" +#include "mesh/mpiCommunications/MPI_iCommData.hpp" +#include "mesh/mpiCommunications/NeighborCommunicator.hpp" + +#include "common/MpiWrapper.hpp" +#include "common/DataTypes.hpp" + +#include +#include +#include +#include + +#include + namespace geos::ghosting { -void doTheNewGlobalNumbering() +struct Exchange +{ + std::set< NodeGlbIdx > nodes; + std::set< Edge > edges; + std::set< Face > faces; +}; + +void to_json( json & j, + const Exchange & v ) +{ + j = json{ { "nodes", v.nodes }, + { "edges", v.edges }, + { "faces", v.faces } }; +} + +void from_json( const json & j, + Exchange & v ) +{ + v.nodes = j.at( "nodes" ).get< std::set< NodeGlbIdx > >(); + v.edges = j.at( "edges" ).get< std::set< Edge > >(); + v.faces = j.at( "faces" ).get< std::set< Face > >(); +} + +array1d< std::uint8_t > convertExchange( Exchange const & exchange ) +{ + std::vector< std::uint8_t > const tmp = json::to_cbor( json( exchange ) ); + array1d< std::uint8_t > result; + result.reserve( tmp.size() ); + for( std::uint8_t const & t: tmp ) + { + result.emplace_back( t ); + } + return result; +} + +Exchange convertExchange( array1d< std::uint8_t > const & input ) +{ + std::vector< std::uint8_t > const tmp( std::cbegin( input ), std::cend( input ) ); + return json::from_cbor( tmp, false ).template get< Exchange >(); +} + +/** + * @brief Extract the cells at the boundary of the mesh. + * @param mesh The vtk mesh. + * @return The vtk cell ids. + */ +std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) { + // TODO Better handle the boundary information, forgetting about the 3d cells and simply handling the outside shell. + auto f = vtkDataSetSurfaceFilter::New(); + f->PassThroughCellIdsOn(); + f->PassThroughPointIdsOff(); + f->FastModeOff(); + + string const originalCellsKey = "ORIGINAL_CELLS"; + f->SetOriginalCellIdsName( originalCellsKey.c_str() ); + auto boundaryMesh = vtkPolyData::New(); + f->UnstructuredGridExecute( mesh, boundaryMesh ); + vtkIdTypeArray const * originalCells = vtkIdTypeArray::FastDownCast( boundaryMesh->GetCellData()->GetArray( originalCellsKey.c_str() ) ); + + std::set< vtkIdType > boundaryCellIdxs; + for( auto i = 0; i < originalCells->GetNumberOfTuples(); ++i ) + { + boundaryCellIdxs.insert( originalCells->GetValue( i ) ); + } + + return boundaryCellIdxs; +} + +Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) +{ + std::size_t const n = nodes.size(); + + // Handles negative values of `i`. + auto const modulo = [n]( integer const & i ) -> std::size_t + { + integer mod = i % n; + if( mod < 0 ) + { + mod += n; + } + return mod; + }; + + Face f; + f.reserve( n ); + + auto const it = std::min_element( nodes.cbegin(), nodes.cend() ); + std::size_t const minIdx = std::distance( nodes.cbegin(), it ); + int const increment = nodes[modulo( minIdx - 1 )] < nodes[modulo( minIdx + 1 )] ? -1 : 1; // TODO based on increment, I can say if the face is flipped or not. + integer i = minIdx; + for( std::size_t count = 0; count < n; ++count, i = i + increment ) + { + f.emplace_back( nodes.at( modulo( i ) ) ); + } + + return f; +} + + +Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, + std::set< vtkIdType > const & cellIds ) +{ + vtkIdTypeArray * gids = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); + Span< vtkIdType > const globalPtIds( (vtkIdType *) gids->GetPointer( 0 ), gids->GetNumberOfTuples() ); + + std::vector< Edge > tmpEdges; + std::vector< Face > tmpFaces; + + // Pre-allocation of the temporary vectors. + { + std::size_t numEdges = 0; + std::size_t numFaces = 0; + for( vtkIdType const & c: cellIds ) + { + vtkCell * cell = mesh->GetCell( c ); + numEdges += cell->GetNumberOfEdges(); + numFaces += cell->GetNumberOfFaces(); + } + tmpEdges.reserve( numEdges ); + tmpFaces.reserve( numFaces ); + } + + // Filling the temporary. + for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) + { + vtkCell * cell = mesh->GetCell( c ); + + for( auto e = 0; e < cell->GetNumberOfEdges(); ++e ) + { + vtkCell * edge = cell->GetEdge( e ); + vtkIdType const ln0 = edge->GetPointId( 0 ); + vtkIdType const ln1 = edge->GetPointId( 1 ); + vtkIdType const gn0 = globalPtIds[ln0]; + vtkIdType const gn1 = globalPtIds[ln1]; + tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ gn0 }, NodeGlbIdx{ gn1 } } ) ); + } + + for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) + { + vtkCell * face = cell->GetFace( f ); + vtkIdList * pids = face->GetPointIds(); + std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); + for( std::size_t i = 0; i < nodes.size(); ++i ) + { + vtkIdType const nli = face->GetPointId( i ); + vtkIdType const ngi = globalPtIds[nli]; + nodes[i] = NodeGlbIdx{ ngi }; + } + tmpFaces.emplace_back( reorderFaceNodes( nodes ) ); + } + } + + std::set< NodeGlbIdx > nodes; + std::transform( std::cbegin( globalPtIds ), std::cend( globalPtIds ), + std::inserter( nodes, std::end( nodes ) ), []( vtkIdType const & id ) + { return NodeGlbIdx{ id }; } ); + + // Removing the duplicates by copying into a `std::set`. + std::set< Edge > edges{ tmpEdges.cbegin(), tmpEdges.cend() }; // SortedArray requires the input to be sorted already. + std::set< Face > faces{ tmpFaces.cbegin(), tmpFaces.cend() }; + + return { std::move( nodes ), std::move( edges ), std::move( faces ) }; +} + + +Exchange buildFullData( vtkSmartPointer< vtkDataSet > mesh ) +{ + std::set< vtkIdType > cellIds; + for( vtkIdType i = 0; i < mesh->GetNumberOfCells(); ++i ) + { + cellIds.insert( cellIds.end(), i ); + } + + return buildSpecificData( mesh, cellIds ); +} + +Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) +{ + return buildSpecificData( mesh, extractBoundaryCells( mesh ) ); +} + +/** + * @brief + * @tparam GLB_IDX The global index type of the geometrical quantity considered (typically @c EdgeGlbIdx or @c FaceGlbIdx). + * @param sizes The sizes of the intersection bucket (from the current MPI rank). + * @param offsets The offsets for each intersection bucket (from the MPI_Scan process, i.e. the previous ranks). + * @param curRank Current MPI rank. + * @return The offset buckets, updated with the sizes of the current MPI rank. + */ +template< typename GLB_IDX > +std::map< std::set< MpiRank >, GLB_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, + std::map< std::set< MpiRank >, GLB_IDX > const & offsets, + MpiRank curRank ) +{ + std::map< std::set< MpiRank >, GLB_IDX > reducedOffsets; + + // We only keep the offsets that are still relevant to the current and higher ranks. + // Note that `reducedOffsets` will be used by the _current_ rank too. + // Therefore, we'll send information that may not be useful to higher ranks. + // This is surely acceptable because the information is tiny. + for( auto const & [ranks, offset]: offsets ) + { + MpiRank const maxConcerned = *std::max_element( ranks.cbegin(), ranks.cend() ); + // TODO invert +// if( maxConcerned >= curRank ) +// { +// reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); +// } + if( maxConcerned < curRank ) + { + continue; + } + reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); + } + + // Add the offsets associated to the new size buckets from the current rank. + GLB_IDX nextOffset{ 0 }; + for( auto const & [ranks, size]: sizes ) + { + auto const it = reducedOffsets.find( ranks ); + if( it == reducedOffsets.end() ) + { + reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, nextOffset ); + nextOffset += GLB_IDX{ size }; + } + else + { + nextOffset = it->second + GLB_IDX{ size }; // Define the new offset from the last + } + } + + // Add an extra entry based for the following rank + reducedOffsets.emplace_hint( reducedOffsets.end(), std::set< MpiRank >{ curRank + 1_mpi }, nextOffset ); + + return reducedOffsets; +} + +// TODO Duplicated +std::map< MpiRank, Exchange > exchange( int commId, + std::vector< NeighborCommunicator > & neighbors, + Exchange const & data ) +{ + MPI_iCommData commData( commId ); + integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); + commData.resize( numNeighbors ); + + + array1d< std::uint8_t > const cv = convertExchange( data ); + + for( integer i = 0; i < numNeighbors; ++i ) + { + neighbors[i].mpiISendReceiveSizes( cv, + commData.mpiSendBufferSizeRequest( i ), + commData.mpiRecvBufferSizeRequest( i ), + commId, + MPI_COMM_GEOSX ); + } + + MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus() ); + MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus() ); + + array1d< array1d< std::uint8_t > > rawExchanged( neighbors.size() ); + + for( integer i = 0; i < numNeighbors; ++i ) + { + neighbors[i].mpiISendReceiveData( cv, + commData.mpiSendBufferRequest( i ), + rawExchanged[i], + commData.mpiRecvBufferRequest( i ), + commId, + MPI_COMM_GEOSX ); + } + MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferRequest(), commData.mpiSendBufferStatus() ); + MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferRequest(), commData.mpiRecvBufferStatus() ); + + std::map< MpiRank, Exchange > output; + for( auto i = 0; i < numNeighbors; ++i ) + { + output[MpiRank{ neighbors[i].neighborRank() }] = convertExchange( rawExchanged[i] ); + } + return output; +} + +/** + * @brief + * @tparam T Typically NodeGloIdx or a container of NodeGlbIdx (for edges and faces) + * @param exchanged + * @param curRank + * @param neighbors + * @return + */ +template< typename T > +std::map< std::set< MpiRank >, std::set< T > > +buildIntersectionBuckets( std::map< MpiRank, std::set< T > const & > const & exchanged, + MpiRank curRank, + std::set< MpiRank > const & neighbors ) +{ + std::map< T, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? + // We "register" all the edges of the current rank: they are the only one we're interested in. + for( T const & node: exchanged.at( curRank ) ) + { + counts.emplace_hint( counts.end(), node, std::set< MpiRank >{ curRank } ); + } + + // We now loop on the neighbor edges. + // If a neighbor has an edge in common with the current rank, they we store it. + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + { + for( T const & node: exchanged.at( neighborRank ) ) + { + auto it = counts.find( node ); + if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. + { + it->second.insert( neighborRank ); + } + } + } + + std::map< std::set< MpiRank >, std::set< T > > nodeBuckets; + for( auto const & [node, ranks]: counts ) + { + if( ranks.find( curRank ) != ranks.cend() ) + { + nodeBuckets[ranks].insert( node ); + } + } + return nodeBuckets; +} + +/** + * @brief Compute the intersection between for the ranks based on the information @p exchanged. + * @param exchanged Geometrical information provided by the ranks (including the current rank). + * @param curRank The current MPI rank. + * @param neighbors excluding current rank + * @return The intersection buckets. + */ +Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, + MpiRank curRank, + std::set< MpiRank > const & neighbors ) +{ + std::map< MpiRank, std::set< NodeGlbIdx > const & > nodeInfo; + for( auto const & [rank, exchange]: exchanged ) + { + nodeInfo.emplace( rank, exchange.nodes ); + } + std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > const nodeBuckets = buildIntersectionBuckets( nodeInfo, curRank, neighbors ); + + std::map< MpiRank, std::set< Edge > const & > edgeInfo; + for( auto const & [rank, exchange]: exchanged ) + { + edgeInfo.emplace( rank, exchange.edges ); + } + std::map< std::set< MpiRank >, std::set< Edge > > const edgeBuckets = buildIntersectionBuckets( edgeInfo, curRank, neighbors ); + + // For faces, the algorithm can be a tad simpler because faces can be shared by at most 2 ranks. + std::map< std::set< MpiRank >, std::set< Face > > faceBuckets; + std::set< Face > curFaces = exchanged.at( curRank ).faces; + for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. + { + for( Face const & face: exchanged.at( neighborRank ).faces ) + { + auto it = curFaces.find( face ); + if( it != curFaces.cend() ) + { + faceBuckets[{ curRank, neighborRank }].insert( *it ); + curFaces.erase( it ); + } + } + } + faceBuckets[{ curRank }] = curFaces; + + return { nodeBuckets, edgeBuckets, faceBuckets }; +} + + +/** + * @brief Estimate an upper bound of the serialized size of the size @p buckets. + * @param buckets + * @return Size in bytes. + * @details @c MPI_Scan requires a fixed buffer size. + * An a-priori estimation of the maximum size of the buffer leads to crazy amounts of memory + * because of the combinatorial pattern of the rank intersections. + * Therefore, we use an initial MPI communication to get a better estimation. + * @node We don't care about the @c nodes because their global numbering is already done. + */ +std::size_t buildMaxBufferSize( Buckets const & buckets ) // TODO give the size buckets instead? +{ + auto const f = []( auto const & bucket ) -> std::size_t + { + std::size_t size = std::size( bucket ); + for( auto const & [ranks, _]: bucket ) + { + size += std::size( ranks ) + 1 + 1; // One `+1` for the size of the ranks set, the other one for the offset + } + return size; + }; + + return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); // Add the ScannedOffsets::{edge, face}Restart +} + +struct BucketSizes +{ + using mapping = std::map< std::set< MpiRank >, localIndex >; + mapping edges; + mapping faces; +}; + +void to_json( json & j, + const BucketSizes & v ) +{ + j = json{ { "edges", v.edges }, + { "faces", v.faces } }; +} + +void from_json( const json & j, + BucketSizes & v ) +{ + v.edges = j.at( "edges" ).get< BucketSizes::mapping >(); + v.faces = j.at( "faces" ).get< BucketSizes::mapping >(); +} + +std::vector< std::uint8_t > serialize( BucketSizes const & sizes ) +{ + return json::to_cbor( json( sizes ) ); +} + +std::vector< std::uint8_t > serialize( BucketOffsets const & offsets ) +{ + return json::to_cbor( json( offsets ) ); +} + +/** + * @brief Compute the sizes of the intersection buckets. + * @param buckets The intersection buckets. + * @return The buckets of sizes. + */ +BucketSizes getBucketSize( Buckets const & buckets ) +{ + BucketSizes output; + + for( auto const & [ranks, edges]: buckets.edges ) + { + output.edges.emplace_hint( output.edges.end(), ranks, std::size( edges ) ); + } + + for( auto const & [ranks, faces]: buckets.faces ) + { + output.faces.emplace_hint( output.faces.end(), ranks, std::size( faces ) ); + } + + return output; +} + +/** + * @brief + * @tparam V Container of std::uint8_t + * @param data + * @return + */ +template< class V > +BucketOffsets deserialize( V const & data ) +{ + return json::from_cbor( data, false ).template get< BucketOffsets >(); +} + +/** + * @brief Custom MPI reduction function. Merges the sizes of the bucket from current rank into numbering offsets. + * @param[in] in Contains the reduced result from the previous ranks. Needs to be unpacked into a @c BucketOffsets instance. + * @param[inout] inout As an @e input, contains a pointer to the @c BucketSizes instance from the current rank. + * As an @e output, will contained the (packed) instance of @c BucketOffsets after the @c reduction. + * The @e output instance will be used by both current and next ranks. + * @param[in] len Number of data. + * @param[in] dataType Type of data. Mean to be @c MPI_BYTE for the current case. + */ +void f( void * in, + void * inout, + int * len, + MPI_Datatype * dataType ) +{ + GEOS_ASSERT_EQ( *dataType, MPI_BYTE ); + + MpiRank const curRank{ MpiWrapper::commRank() }; + + // offsets provided by the previous rank(s) + BucketOffsets const offsets = deserialize( Span< std::uint8_t >( (std::uint8_t *) in, *len ) ); + + // Sizes provided by the current rank, under the form of a pointer to the data. + // No need to serialize, since we're on the same rank. + std::uintptr_t addr; + std::memcpy( &addr, inout, sizeof( std::uintptr_t ) ); + BucketSizes const * sizes = reinterpret_cast(addr); + + BucketOffsets updatedOffsets; + updatedOffsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes->edges, offsets.edges, curRank ); + updatedOffsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes->faces, offsets.faces, curRank ); + + // Serialize the updated offsets, so they get sent to the next rank. + std::vector< std::uint8_t > const serialized = serialize( updatedOffsets ); + std::memcpy( inout, serialized.data(), serialized.size() ); +} + +std::tuple< Buckets, BucketOffsets > doTheNewGlobalNumbering( vtkSmartPointer< vtkDataSet > mesh, + std::set< MpiRank > const & neighbors ) +{ + // Now we exchange the data with our neighbors. + MpiRank const curRank{ MpiWrapper::commRank() }; + + std::vector< NeighborCommunicator > ncs; + ncs.reserve( neighbors.size() ); + for( MpiRank const & rank: neighbors ) + { + ncs.emplace_back( rank.get() ); + } + + CommID const commId = CommunicationTools::getInstance().getCommID(); + std::map< MpiRank, Exchange > exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); + exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); + +// std::cout << "exchanged on rank " << curRank << " -> " << json( exchanged[MpiRank{ MpiWrapper::commRank() }] ) << std::endl; + + Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); + + std::size_t const maxBufferSize = 100 * buildMaxBufferSize( buckets ); + + std::vector< std::uint8_t > sendBuffer( maxBufferSize ); + std::vector< std::uint8_t > recvBuffer( maxBufferSize ); + + BucketSizes const sizes = getBucketSize( buckets ); + BucketOffsets offsets; + if( curRank == 0_mpi ) + { + // The `MPI_Scan` process will not call the reduction operator for rank 0. + // So we need to reduce ourselves for ourselves. + offsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes.edges, { { { 0_mpi, }, 0_egi } }, curRank ); + offsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes.faces, { { { 0_mpi, }, 0_fgi } }, curRank ); + // Still we need to send this reduction to the following rank, by copying to it to the send buffer. + std::vector< std::uint8_t > const bytes = serialize( offsets ); + std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); + } + else + { + // For the other ranks, the reduction operator will be called during the `Mpi_Scan` process. + // So unlike for rank 0, we do not have to do it ourselves. + // In order to provide the `sizes` to the reduction operator, + // since `sizes` will only be used on the current rank, + // we'll provide the information as a pointer to the instance. + // The reduction operator will then compute the new offsets and send them to the following rank. + std::uintptr_t const addr = reinterpret_cast(&sizes); + std::memcpy( sendBuffer.data(), &addr, sizeof( std::uintptr_t ) ); + } + + MPI_Op op; + MPI_Op_create( f, false, &op ); + + MPI_Scan( sendBuffer.data(), recvBuffer.data(), maxBufferSize, MPI_BYTE, op, MPI_COMM_WORLD ); + + if( curRank != 0_mpi ) + { + offsets = deserialize( recvBuffer ); + } + return { std::move( buckets ), std::move( offsets ) }; } } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp index fd1db1078a9..ae83b271830 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp @@ -15,10 +15,75 @@ #ifndef GEOS_NEWGLOBALNUMBERING_HPP #define GEOS_NEWGLOBALNUMBERING_HPP +#include "Indices.hpp" + +#include +#include + +#include + namespace geos::ghosting { -void doTheNewGlobalNumbering(); +using Edge = std::tuple< NodeGlbIdx, NodeGlbIdx >; +using Face = std::vector< NodeGlbIdx >; + +struct Buckets +{ + std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > nodes; + std::map< std::set< MpiRank >, std::set< Edge > > edges; + std::map< std::set< MpiRank >, std::set< Face > > faces; +}; + +inline void to_json( json & j, + const Buckets & v ) +{ + j = json{ { "nodes", v.nodes }, + { "edges", v.edges }, + { "faces", v.faces } }; +} + +struct BucketOffsets +{ + std::map< std::set< MpiRank >, EdgeGlbIdx > edges; + std::map< std::set< MpiRank >, FaceGlbIdx > faces; +}; + +inline void to_json( json & j, + const BucketOffsets & v ) +{ + j = json{ { "edges", v.edges }, + { "faces", v.faces } }; +} + +inline void from_json( const json & j, + BucketOffsets & v ) +{ + v.edges = j.at( "edges" ).get< std::map< std::set< MpiRank >, EdgeGlbIdx > >(); + v.faces = j.at( "faces" ).get< std::map< std::set< MpiRank >, FaceGlbIdx > >(); +} + +/** + * @brief Order the nodes of the faces in a way that can be reproduced across the MPI ranks. + * @param nodes The list of nodes as provided by the mesh. + * @return A face with the nodes in the appropriate order + * @details The nodes will be ordered in the following way. + * First, we look for the lowest node index. It will become the first node. + * Then we must pick the second node. We have to choices: just before or just after the first. + * (we do not want to shuffle the nodes completely, we need to keep track of the order). + * To do this, we select the nodes with the lowest index as well. + * This also defines a direction in which we'll pick the other nodes. + * For example, the face [2, 3, 1, 5, 9, 8] will become [1, 3, 2, 8, 9, 5] + * because we'll start with @c 1 and then select the @c 3 over the @c 5. + * Which defines the direction @c 2, @c 8, @c 9, @c 5. + * @note This is the same pattern that we apply for edges. + * Except that edges having only two nodes, it's not necessary to implement a dedicated function + * and std::minmax is enough. + */ +Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ); + +std::tuple< Buckets, BucketOffsets > doTheNewGlobalNumbering( vtkSmartPointer< vtkDataSet > mesh, + std::set< MpiRank > const & neighbors ); } // end of namespace From 0d91faede00901a5ed73a33beeb26b23f6593ca5 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 18:17:22 -0700 Subject: [PATCH 071/106] Remove useless identity matrix --- src/coreComponents/mesh/generators/NewGhosting.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index fd6f89947fc..f6837575d56 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -545,13 +545,6 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostInfo.NumGlobalRows() ); } - Epetra_CrsMatrix identity( Epetra_DataAccess::Copy, ownedMap, 1, true ); - for( int const & i: ownedGlbIdcs ) - { - identity.InsertGlobalValues( i, 1, ones.data(), &i ); - } - identity.FillComplete(); - auto tDownward = makeTranspose( upward ); // TODO give it to multiply! // EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-before.mat", *tDownward ); Epetra_Vector const zeros( ownedMap, true ); From b3d911c3b5ed9685a1187063df9e4a3859ad2c12 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 20 May 2024 21:10:10 -0700 Subject: [PATCH 072/106] Finish the class that manages the matrix index <-> geometry index conversion. --- .../mesh/generators/NewGhosting.cpp | 134 ++++++++++-------- 1 file changed, 77 insertions(+), 57 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index f6837575d56..618a2aae523 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -319,15 +319,22 @@ class FindGeometricalType CELL }; - FindGeometricalType( std::size_t const edgeOffset, - std::size_t const faceOffset, - std::size_t const cellOffset ) - : m_edgeOffset( intConv< int >( edgeOffset ) ), - m_faceOffset( intConv< int >( faceOffset ) ), - m_cellOffset( intConv< int >( cellOffset ) ) + FindGeometricalType( NodeGlbIdx const & maxNodeGlbIdx, + EdgeGlbIdx const & maxEdgeGlbIdx, + FaceGlbIdx const & maxFaceGlbIdx, + CellGlbIdx const & maxCellGlbIdx ) + : m_edgeOffset( intConv< int >( maxNodeGlbIdx.get() + 1 ) ), + m_faceOffset( intConv< int >( m_edgeOffset + maxEdgeGlbIdx.get() + 1 ) ), + m_cellOffset( intConv< int >( m_faceOffset + maxFaceGlbIdx.get() + 1 ) ), + m_numEntries( intConv< int >( m_cellOffset + maxCellGlbIdx.get() + 1 ) ) { } - Geom operator()( int const & index ) const + [[nodiscard]] int numEntries() const + { + return m_numEntries; + } + + [[nodiscard]] Geom getGeometricalType( int const & index ) const { if( index < m_edgeOffset ) { @@ -347,35 +354,51 @@ class FindGeometricalType } } - template< typename GLOBAL_INDEX > - [[nodiscard]] GLOBAL_INDEX as( int const & index ) const + [[nodiscard]] NodeGlbIdx toNodeGlbIdx( int const & index ) const { - if constexpr( std::is_same_v< GLOBAL_INDEX, NodeGlbIdx > ) - { - return NodeGlbIdx{ intConv< NodeGlbIdx::UnderlyingType >( index ) }; - } - else if constexpr( std::is_same_v< GLOBAL_INDEX, EdgeGlbIdx > ) - { - return EdgeGlbIdx{ intConv< EdgeGlbIdx::UnderlyingType >( index - m_edgeOffset ) }; - } - else if constexpr( std::is_same_v< GLOBAL_INDEX, FaceGlbIdx > ) - { - return FaceGlbIdx{ intConv< FaceGlbIdx::UnderlyingType >( index - m_faceOffset ) }; - } - else if constexpr( std::is_same_v< GLOBAL_INDEX, CellGlbIdx > ) - { - return CellGlbIdx{ intConv< CellGlbIdx::UnderlyingType >( index - m_cellOffset ) }; - } - else - { - GEOS_ERROR( "Internal Error." ); - } + return NodeGlbIdx{ intConv< NodeGlbIdx::UnderlyingType >( index ) }; + } + + [[nodiscard]] EdgeGlbIdx toEdgeGlbIdx( int const & index ) const + { + return EdgeGlbIdx{ intConv< EdgeGlbIdx::UnderlyingType >( index - m_edgeOffset ) }; + } + + [[nodiscard]] FaceGlbIdx toFaceGlbIdx( int const & index ) const + { + return FaceGlbIdx{ intConv< FaceGlbIdx::UnderlyingType >( index - m_faceOffset ) }; + } + + [[nodiscard]] CellGlbIdx toCellGlbIdx( int const & index ) const + { + return CellGlbIdx{ intConv< CellGlbIdx::UnderlyingType >( index - m_cellOffset ) }; + } + + [[nodiscard]] int fromNodeGlbIdx( NodeGlbIdx const & ngi ) const + { + return intConv< int >( ngi.get() ); + } + + [[nodiscard]] int fromEdgeGlbIdx( EdgeGlbIdx const & egi ) const + { + return intConv< int >( egi.get() + m_edgeOffset ); + } + + [[nodiscard]] int fromFaceGlbIdx( FaceGlbIdx const & fgi ) const + { + return intConv< int >( fgi.get() + m_faceOffset ); + } + + [[nodiscard]] int fromCellGlbIdx( CellGlbIdx const & cgi ) const + { + return intConv< int >( cgi.get() + m_cellOffset ); } private: int const m_edgeOffset; int const m_faceOffset; int const m_cellOffset; + int const m_numEntries; }; std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & owned, @@ -383,11 +406,10 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o MaxGlbIdcs const & gis, MpiRank curRank ) { - std::size_t const edgeOffset = gis.nodes.get() + 1; - std::size_t const faceOffset = edgeOffset + gis.edges.get() + 1; - std::size_t const cellOffset = faceOffset + gis.faces.get() + 1; + FindGeometricalType const convert( gis.nodes, gis.edges, gis.faces, gis.cells ); + using Geom = FindGeometricalType::Geom; - std::size_t const n = cellOffset + gis.cells.get() + 1; // Total number of entries in the graph. + std::size_t const n = convert.numEntries(); // Total number of entries in the graph. std::size_t const numOwnedNodes = std::size( owned.n ); std::size_t const numOwnedEdges = std::size( owned.e2n ); @@ -410,21 +432,21 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o otherGlbIdcs.reserve( numOther ); for( NodeGlbIdx const & ngi: present.n ) { - otherGlbIdcs.emplace_back( ngi.get() ); + otherGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); } for( auto const & [egi, _]: present.e2n ) { - otherGlbIdcs.emplace_back( egi.get() + edgeOffset ); + otherGlbIdcs.emplace_back( convert.fromEdgeGlbIdx( egi ) ); } for( auto const & [fgi, _]: present.f2e ) { - otherGlbIdcs.emplace_back( fgi.get() + faceOffset ); + otherGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); } GEOS_ASSERT_EQ( numOther, std::size( otherGlbIdcs ) ); for( NodeGlbIdx const & ngi: owned.n ) { - auto const i = ngi.get(); + int const i = convert.fromNodeGlbIdx( ngi ); ownedGlbIdcs.emplace_back( i ); numEntriesPerRow.emplace_back( 1 ); std::vector< int > const tmp( 1, ownedGlbIdcs.back() ); @@ -432,7 +454,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } for( auto const & [egi, nodes]: owned.e2n ) { - auto const i = egi.get() + edgeOffset; + int const i = convert.fromEdgeGlbIdx( egi ); ownedGlbIdcs.emplace_back( i ); numEntriesPerRow.emplace_back( std::tuple_size_v< decltype( nodes ) > + 1 ); // `+1` comes from the identity std::vector< int > const tmp{ int( std::get< 0 >( nodes ).get() ), int( std::get< 1 >( nodes ).get() ), ownedGlbIdcs.back() }; @@ -440,28 +462,28 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } for( auto const & [fgi, edges]: owned.f2e ) { - auto const i = fgi.get() + faceOffset; + int const i = convert.fromFaceGlbIdx( fgi ); ownedGlbIdcs.emplace_back( i ); numEntriesPerRow.emplace_back( std::size( edges ) + 1 ); // `+1` comes from the identity std::vector< int > tmp; tmp.reserve( numEntriesPerRow.back() ); for( EdgeGlbIdx const & egi: edges ) { - tmp.emplace_back( egi.get() + edgeOffset ); + tmp.emplace_back( convert.fromEdgeGlbIdx( egi ) ); } tmp.emplace_back( ownedGlbIdcs.back() ); indices.emplace_back( tmp ); } for( auto const & [cgi, faces]: owned.c2f ) { - auto const i = cgi.get() + cellOffset; + int const i = convert.fromCellGlbIdx( cgi ); ownedGlbIdcs.emplace_back( i ); numEntriesPerRow.emplace_back( std::size( faces ) + 1 ); // `+1` comes from the identity std::vector< int > tmp; tmp.reserve( numEntriesPerRow.back() ); for( FaceGlbIdx const & fgi: faces ) { - tmp.emplace_back( fgi.get() + faceOffset ); + tmp.emplace_back( convert.fromFaceGlbIdx( fgi ) ); } tmp.emplace_back( ownedGlbIdcs.back() ); indices.emplace_back( tmp ); @@ -565,8 +587,6 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o // int extractedEdges = 0; // std::vector< double > extractedEdgesValues( n ); // std::vector< int > extractedEdgesIndices( n ); - FindGeometricalType const getGeomType( edgeOffset, faceOffset, cellOffset ); - using Geom = FindGeometricalType::Geom; Ownerships ownerships; for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? @@ -579,17 +599,17 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o ownerships.neighbors.insert( owner ); } - switch( getGeomType( index ) ) + switch( convert.getGeometricalType( index ) ) { case Geom::NODE: { - NodeGlbIdx const ngi = getGeomType.as< NodeGlbIdx >( index ); + NodeGlbIdx const ngi = convert.toNodeGlbIdx( index ); ownerships.nodes.emplace( ngi, owner ); break; } case Geom::EDGE: { - EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); + EdgeGlbIdx const egi = convert.toEdgeGlbIdx( index ); ownerships.edges.emplace( egi, owner ); // TODO make all the following check in on time with sets comparison instead of "point-wise" comparisons. // TODO same for the faces and cells... @@ -601,7 +621,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } case Geom::FACE: { - FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); + FaceGlbIdx const fgi = convert.toFaceGlbIdx( index ); ownerships.faces.emplace( fgi, owner ); if( owned.f2e.find( fgi ) == owned.f2e.cend() and present.f2e.find( fgi ) == present.f2e.cend() ) { @@ -611,7 +631,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } case Geom::CELL: { - CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); + CellGlbIdx const cgi = convert.toCellGlbIdx( index ); ownerships.cells.emplace( cgi, owner ); if( owned.c2f.find( cgi ) == owned.c2f.cend() ) { @@ -670,36 +690,36 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o // GEOS_LOG( "ext 1 = " << std::size( s1 ) ); // GEOS_LOG_RANK( "ext = " << ext ); int const index = missingIndices[i]; - switch( getGeomType( index ) ) + switch( convert.getGeometricalType( index ) ) { case Geom::EDGE: { GEOS_ASSERT_EQ( ext, 2 ); // TODO check that val != 0? - EdgeGlbIdx const egi = getGeomType.as< EdgeGlbIdx >( index ); - NodeGlbIdx const ngi0 = getGeomType.as< NodeGlbIdx >( extIndices[0] ); - NodeGlbIdx const ngi1 = getGeomType.as< NodeGlbIdx >( extIndices[1] ); + EdgeGlbIdx const egi = convert.toEdgeGlbIdx( index ); + NodeGlbIdx const ngi0 = convert.toNodeGlbIdx( extIndices[0] ); + NodeGlbIdx const ngi1 = convert.toNodeGlbIdx( extIndices[1] ); ghosts.e2n[egi] = std::minmax( { ngi0, ngi1 } ); break; } case Geom::FACE: { GEOS_ASSERT_EQ( ext, 4 ); // TODO temporary check for the dev - FaceGlbIdx const fgi = getGeomType.as< FaceGlbIdx >( index ); + FaceGlbIdx const fgi = convert.toFaceGlbIdx( index ); std::set< EdgeGlbIdx > & tmp = ghosts.f2e[fgi]; for( int ii = 0; ii < ext; ++ii ) { - tmp.insert( EdgeGlbIdx{ intConv< globalIndex >( extIndices[ii] - edgeOffset ) } ); + tmp.insert( convert.toEdgeGlbIdx( extIndices[ii] ) ); } break; } case Geom::CELL: { GEOS_ASSERT_EQ( ext, 6 ); // TODO temporary check for the dev - CellGlbIdx const cgi = getGeomType.as< CellGlbIdx >( index ); + CellGlbIdx const cgi = convert.toCellGlbIdx( index ); std::set< FaceGlbIdx > & tmp = ghosts.c2f[cgi]; for( int ii = 0; ii < ext; ++ii ) { - tmp.insert( FaceGlbIdx{ intConv< globalIndex >( extIndices[ii] - faceOffset ) } ); + tmp.insert( convert.toFaceGlbIdx( extIndices[ii] ) ); } break; } From ec7f9906d41771634ad22c5e34c55915cc70226b Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 22 May 2024 13:47:52 -0700 Subject: [PATCH 073/106] Mainly comments --- .../mesh/generators/NewGhosting.cpp | 80 ++++++++++++------- .../mesh/generators/NewGlobalNumbering.cpp | 10 +-- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 618a2aae523..77712897ea4 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -500,6 +500,8 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); Epetra_Map const ownedMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); + // The `upward` matrix offers a representation of the graph connections. + // The matrix is square. Each size being the number of geometrical entities. Epetra_CrsMatrix upward( Epetra_DataAccess::Copy, ownedMap, numEntriesPerRow.data(), true ); for( std::size_t i = 0; i < numOwned; ++i ) @@ -517,7 +519,11 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o int const commSize( MpiWrapper::commSize() ); // Now let's build the domain indicator matrix. - // It's rectangular, one dimension being the number of MPI ranks, the other the number of nodes in the mesh graph. + // It's rectangular, number of rows being the number of MPI ranks, + // number of columns being the number of nodes in the mesh graph, + // ie the number of geometrical entities in the mesh. + // It contains one for every time the geometrical entity is present on the rank. + // By present, we do not meant that the ranks owns it: just that it's already available on the rank. Epetra_Map const mpiMap( commSize, 0, comm ); // Let the current rank get the appropriate index in this map. Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, mpiMap, numOwned + numOther, true ); @@ -526,6 +532,10 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o indicator.InsertGlobalValues( curRank.get(), numOther, ones.data(), otherGlbIdcs.data() ); indicator.FillComplete( ownedMap, mpiMap ); + // The `ownership` matrix is a diagonal square matrix. + // Each size being the number of geometrical entities. + // The value of the diagonal term will be the owning rank. + // By means of matrices multiplications, it will be possible to exchange the ownerships information across the ranks. // TODO Could we use an Epetra_Vector as a diagonal matrix? std::vector< double > myRank( 1, curRank.get() ); Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, ownedMap, 1, true ); @@ -546,32 +556,38 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); - Epetra_CrsMatrix ghosted( multiply( commSize, indicator, upward ) ); - ghosted.PutScalar( 1. ); // This can be done after the `FillComplete`. - ghosted.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted.mat", ghosted ); + // The `ghostingFootprint` matrix is rectangular, number of columns being the number of MPI ranks, + // the number of rows being the number of nodes in the mesh graph. + // For any MPI rank (ie column), a non-zero entry means + // that the corresponding geometrical entry has to be eventually present onto the rank + // for the ghosting to be effective. + Epetra_CrsMatrix ghostingFootprint( multiply( commSize, indicator, upward ) ); + ghostingFootprint.PutScalar( 1. ); // This can be done after the `FillComplete`. + ghostingFootprint.FillComplete( mpiMap, ownedMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostingFootprint.mat", ghostingFootprint ); + // We put `1` everywhere there's a non-zero entry, so we'll be able to compose with the `ownership` matrix. + + // FIXME TODO WARNING From `ghostingFootprint` extract where I have to send + // Then, perform the ghostingFootprint * ownership matrix multiplication, + // so I get to know from whom I'll receive. if( curRank == 0_mpi ) { - GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghosted.NumGlobalCols() ); - GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghosted.NumGlobalRows() ); + GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghostingFootprint.NumGlobalCols() ); + GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghostingFootprint.NumGlobalRows() ); } - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghosted-after.mat", ghosted ); - Epetra_CrsMatrix ghostInfo( Epetra_DataAccess::Copy, mpiMap, 1, false ); - EpetraExt::MatrixMatrix::Multiply( ghosted, true, ownership, false, ghostInfo, false ); - ghostInfo.FillComplete( ownedMap, mpiMap ); + Epetra_CrsMatrix ghostExchange( Epetra_DataAccess::Copy, mpiMap, 1, false ); + EpetraExt::MatrixMatrix::Multiply( ghostingFootprint, true, ownership, false, ghostExchange, false ); + ghostExchange.FillComplete( ownedMap, mpiMap ); if( curRank == 0_mpi ) { - GEOS_LOG_RANK( "ghostInfo->NumGlobalCols() = " << ghostInfo.NumGlobalCols() ); - GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostInfo.NumGlobalRows() ); + GEOS_LOG_RANK( "ghostInfo->NumGlobalCols() = " << ghostExchange.NumGlobalCols() ); + GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostExchange.NumGlobalRows() ); } - - auto tDownward = makeTranspose( upward ); // TODO give it to multiply! -// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-before.mat", *tDownward ); - Epetra_Vector const zeros( ownedMap, true ); - tDownward->ReplaceDiagonalValues( zeros ); // TODO directly build zeros in the function call. -// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-after.mat", *tDownward ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostInfo.mat", ghostingFootprint ); + std::unique_ptr< Epetra_CrsMatrix > tGhostExchange = makeTranspose( ghostExchange ); + tGhostExchange->FillComplete( mpiMap, ownedMap ); int extracted = 0; std::vector< double > extractedValues( n ); @@ -581,8 +597,12 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o extractedIndices.resize( extracted ); GEOS_LOG_RANK( "extracted = " << extracted ); - std::vector< int > missingIndices; - missingIndices.reserve( n ); + // The `notPresentIndices` container will hold all the graph nodes indices + // that are not already present on the current rank. + // For those indices, will have to gather some additional mapping information, + // since they were not provided through the vtk mesh. + std::vector< int > notPresentIndices; + notPresentIndices.reserve( n ); // int extractedEdges = 0; // std::vector< double > extractedEdgesValues( n ); @@ -615,7 +635,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o // TODO same for the faces and cells... if( owned.e2n.find( egi ) == owned.e2n.cend() and present.e2n.find( egi ) == present.e2n.cend() ) { - missingIndices.emplace_back( index ); + notPresentIndices.emplace_back( index ); } break; } @@ -625,7 +645,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o ownerships.faces.emplace( fgi, owner ); if( owned.f2e.find( fgi ) == owned.f2e.cend() and present.f2e.find( fgi ) == present.f2e.cend() ) { - missingIndices.emplace_back( index ); + notPresentIndices.emplace_back( index ); } break; } @@ -635,7 +655,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o ownerships.cells.emplace( cgi, owner ); if( owned.c2f.find( cgi ) == owned.c2f.cend() ) { - missingIndices.emplace_back( index ); + notPresentIndices.emplace_back( index ); } break; } @@ -648,7 +668,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o GEOS_LOG( "Gathering the missing mappings." ); - std::size_t const numMissingIndices = std::size( missingIndices ); + std::size_t const numMissingIndices = std::size( notPresentIndices ); std::size_t const numGlobalMissingIndices = MpiWrapper::sum( numMissingIndices ); // Epetra_Map const missingIndicesMap( intConv< long long int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); @@ -661,11 +681,17 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o Epetra_CrsMatrix missing( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); for( std::size_t i = 0; i < numMissingIndices; ++i ) { - missing.InsertGlobalValues( offset + i, 1, ones.data(), &missingIndices[i] ); + missing.InsertGlobalValues( offset + i, 1, ones.data(), ¬PresentIndices[i] ); } missing.FillComplete( ownedMap, missingIndicesMap ); // TODO Don't put ones in downward: reassemble with the ordering (e.g. edges point to a first and last node) + + auto tDownward = makeTranspose( upward ); // TODO give it to multiply! +// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-before.mat", *tDownward ); + Epetra_Vector const zeros( ownedMap, true ); + tDownward->ReplaceDiagonalValues( zeros ); // TODO directly build zeros in the function call. +// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-after.mat", *tDownward ); Epetra_CrsMatrix missingMappings( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( missing, false, *tDownward, true, missingMappings, false ); missingMappings.FillComplete( ownedMap, missingIndicesMap ); @@ -689,7 +715,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o // GEOS_LOG( "ext 0 = " << std::size( s0 ) ); // GEOS_LOG( "ext 1 = " << std::size( s1 ) ); // GEOS_LOG_RANK( "ext = " << ext ); - int const index = missingIndices[i]; + int const index = notPresentIndices[i]; switch( convert.getGeometricalType( index ) ) { case Geom::EDGE: diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp index d82e8890be5..cef24c79837 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -162,11 +162,11 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, for( auto e = 0; e < cell->GetNumberOfEdges(); ++e ) { vtkCell * edge = cell->GetEdge( e ); - vtkIdType const ln0 = edge->GetPointId( 0 ); - vtkIdType const ln1 = edge->GetPointId( 1 ); - vtkIdType const gn0 = globalPtIds[ln0]; - vtkIdType const gn1 = globalPtIds[ln1]; - tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ gn0 }, NodeGlbIdx{ gn1 } } ) ); + vtkIdType const nli0 = edge->GetPointId( 0 ); + vtkIdType const nli1 = edge->GetPointId( 1 ); + vtkIdType const ngi0 = globalPtIds[nli0]; + vtkIdType const ngi1 = globalPtIds[nli1]; + tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ ngi0 }, NodeGlbIdx{ ngi1 } } ) ); } for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) From dadc03b279889677b4eea6d7afcefd4b14199461 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 22 May 2024 18:15:07 -0700 Subject: [PATCH 074/106] Send and receive ghost should be OK. --- .../mesh/generators/NewGhosting.cpp | 255 +++++++++++------- 1 file changed, 161 insertions(+), 94 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 77712897ea4..e73cdce710d 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -254,15 +254,41 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, return ptr; } -struct Ownerships +struct GhostSend +{ + std::map< NodeGlbIdx, std::set< MpiRank > > nodes; + std::map< EdgeGlbIdx, std::set< MpiRank > > edges; + std::map< FaceGlbIdx, std::set< MpiRank > > faces; + std::map< CellGlbIdx, std::set< MpiRank > > cells; +}; + +void to_json( json & j, + const GhostSend & v ) +{ + j = json{ { "nodes", v.nodes }, + { "edges", v.edges }, + { "faces", v.faces }, + { "cells", v.cells } }; +} + +struct GhostRecv { std::map< NodeGlbIdx, MpiRank > nodes; std::map< EdgeGlbIdx, MpiRank > edges; std::map< FaceGlbIdx, MpiRank > faces; std::map< CellGlbIdx, MpiRank > cells; - std::set< MpiRank > neighbors; }; +void to_json( json & j, + const GhostRecv & v ) +{ + j = json{ { "nodes", v.nodes }, + { "edges", v.edges }, + { "faces", v.faces }, + { "cells", v.cells } }; +} + + Epetra_CrsMatrix multiply( int commSize, Epetra_CrsMatrix const & indicator, Epetra_CrsMatrix & upward ) @@ -401,10 +427,10 @@ class FindGeometricalType int const m_numEntries; }; -std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & owned, - MeshGraph const & present, - MaxGlbIdcs const & gis, - MpiRank curRank ) +std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph const & owned, + MeshGraph const & present, + MaxGlbIdcs const & gis, + MpiRank curRank ) { FindGeometricalType const convert( gis.nodes, gis.edges, gis.faces, gis.cells ); using Geom = FindGeometricalType::Geom; @@ -442,6 +468,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o { otherGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); } + std::sort( std::begin( otherGlbIdcs ), std::end( otherGlbIdcs ) ); GEOS_ASSERT_EQ( numOther, std::size( otherGlbIdcs ) ); for( NodeGlbIdx const & ngi: owned.n ) @@ -488,6 +515,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o tmp.emplace_back( ownedGlbIdcs.back() ); indices.emplace_back( tmp ); } + std::sort( std::begin( ownedGlbIdcs ), std::end( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, std::size( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, std::size( numEntriesPerRow ) ); @@ -556,7 +584,8 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); - // The `ghostingFootprint` matrix is rectangular, number of columns being the number of MPI ranks, + // The `ghostingFootprint` matrix is rectangular, + // the number of columns being the number of MPI ranks, // the number of rows being the number of nodes in the mesh graph. // For any MPI rank (ie column), a non-zero entry means // that the corresponding geometrical entry has to be eventually present onto the rank @@ -580,108 +609,149 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o Epetra_CrsMatrix ghostExchange( Epetra_DataAccess::Copy, mpiMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( ghostingFootprint, true, ownership, false, ghostExchange, false ); ghostExchange.FillComplete( ownedMap, mpiMap ); + if( curRank == 0_mpi ) { - GEOS_LOG_RANK( "ghostInfo->NumGlobalCols() = " << ghostExchange.NumGlobalCols() ); - GEOS_LOG_RANK( "ghostInfo->NumGlobalRows() = " << ghostExchange.NumGlobalRows() ); + GEOS_LOG_RANK( "ghostExchange->NumGlobalCols() = " << ghostExchange.NumGlobalCols() ); + GEOS_LOG_RANK( "ghostExchange->NumGlobalRows() = " << ghostExchange.NumGlobalRows() ); } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostInfo.mat", ghostingFootprint ); - std::unique_ptr< Epetra_CrsMatrix > tGhostExchange = makeTranspose( ghostExchange ); - tGhostExchange->FillComplete( mpiMap, ownedMap ); + + int extracted2 = 0; + std::vector< double > extractedValues2( commSize ); + std::vector< int > extractedIndices2( commSize ); + + GhostSend send; + for( int const & index: ownedGlbIdcs ) + { + ghostingFootprint.ExtractGlobalRowCopy( index, commSize, extracted2, extractedValues2.data(), extractedIndices2.data() ); + std::set< MpiRank > * sendingTo = nullptr; + switch( convert.getGeometricalType( index ) ) + { + case Geom::NODE: + { + sendingTo = &send.nodes[convert.toNodeGlbIdx( index )]; + break; + } + case Geom::EDGE: + { + sendingTo = &send.edges[convert.toEdgeGlbIdx( index )]; + break; + } + case Geom::FACE: + { + sendingTo = &send.faces[convert.toFaceGlbIdx( index )]; + break; + } + case Geom::CELL: + { + sendingTo = &send.cells[convert.toCellGlbIdx( index )]; + break; + } + } + + for( int ii = 0; ii < extracted2; ++ii ) + { + MpiRank const rank{ extractedIndices2[ii] }; + if( rank != curRank ) + { + sendingTo->insert( rank ); + } + } + } int extracted = 0; std::vector< double > extractedValues( n ); std::vector< int > extractedIndices( n ); - ghostInfo.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); + ghostExchange.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); extractedValues.resize( extracted ); extractedIndices.resize( extracted ); - GEOS_LOG_RANK( "extracted = " << extracted ); - - // The `notPresentIndices` container will hold all the graph nodes indices - // that are not already present on the current rank. - // For those indices, will have to gather some additional mapping information, - // since they were not provided through the vtk mesh. - std::vector< int > notPresentIndices; - notPresentIndices.reserve( n ); - -// int extractedEdges = 0; -// std::vector< double > extractedEdgesValues( n ); -// std::vector< int > extractedEdgesIndices( n ); - Ownerships ownerships; - for( int i = 0; i < extracted; ++i ) // TODO Can we `zip` instead? + std::set< int > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); + std::set< int > receivedIndices; // The indices my neighbors will send me. + std::set_difference( std::cbegin( allNeededIndices ), std::cend( allNeededIndices ), + std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ), + std::inserter( receivedIndices, std::end( receivedIndices ) ) ); + std::vector< int > notPresentIndices_; // TODO remove _ + std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), + std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), + std::back_inserter( notPresentIndices_ ) ); +// { +// std::set< int > const tmp( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); +// GEOS_LOG_RANK( "tmp = " << json( tmp ) ); +// GEOS_LOG_RANK( "receivedIndices = " << json( receivedIndices ) ); +// GEOS_ASSERT( tmp == receivedIndices ); +// } +// GEOS_LOG_RANK( "extracted = " << extracted ); +// if( curRank == 1_mpi ) +// { +// GEOS_LOG_RANK( "extractedIndices = " << json( extractedIndices ) ); +//// GEOS_LOG_RANK( "extracted values = " << json( extractedValues ) ); +// GEOS_LOG_RANK( "receivedIndices = " << json( receivedIndices ) ); +// } + GhostRecv recv; + for( int i = 0; i < extracted; ++i ) +// for( int const & receivedIndex: receivedIndices ) { int const & index = extractedIndices[i]; - - MpiRank const owner = MpiRank{ int( extractedValues[i] ) }; - if( owner != curRank ) + if( receivedIndices.find( index ) == std::cend( receivedIndices ) ) // TODO make a map `receivedIndices -> mpi rank` { - ownerships.neighbors.insert( owner ); + continue; } + MpiRank const sender{ int( extractedValues[i] ) }; switch( convert.getGeometricalType( index ) ) { case Geom::NODE: { - NodeGlbIdx const ngi = convert.toNodeGlbIdx( index ); - ownerships.nodes.emplace( ngi, owner ); + recv.nodes[convert.toNodeGlbIdx( index )] = sender; break; } case Geom::EDGE: { - EdgeGlbIdx const egi = convert.toEdgeGlbIdx( index ); - ownerships.edges.emplace( egi, owner ); - // TODO make all the following check in on time with sets comparison instead of "point-wise" comparisons. - // TODO same for the faces and cells... - if( owned.e2n.find( egi ) == owned.e2n.cend() and present.e2n.find( egi ) == present.e2n.cend() ) - { - notPresentIndices.emplace_back( index ); - } + recv.edges[convert.toEdgeGlbIdx( index )] = sender; break; } case Geom::FACE: { - FaceGlbIdx const fgi = convert.toFaceGlbIdx( index ); - ownerships.faces.emplace( fgi, owner ); - if( owned.f2e.find( fgi ) == owned.f2e.cend() and present.f2e.find( fgi ) == present.f2e.cend() ) - { - notPresentIndices.emplace_back( index ); - } + recv.faces[convert.toFaceGlbIdx( index )] = sender; break; } case Geom::CELL: { - CellGlbIdx const cgi = convert.toCellGlbIdx( index ); - ownerships.cells.emplace( cgi, owner ); - if( owned.c2f.find( cgi ) == owned.c2f.cend() ) - { - notPresentIndices.emplace_back( index ); - } + recv.cells[convert.toCellGlbIdx( index )] = sender; break; } - default: - { - GEOS_ERROR( "Internal error" ); - } } } - GEOS_LOG( "Gathering the missing mappings." ); +// if( curRank == 1_mpi or curRank == 2_mpi ) +// { +// GEOS_LOG_RANK( "recv.edges = " << json( recv.edges ) ); +// GEOS_LOG_RANK( "send.edges = " << json( send.edges ) ); +// } - std::size_t const numMissingIndices = std::size( notPresentIndices ); - std::size_t const numGlobalMissingIndices = MpiWrapper::sum( numMissingIndices ); + // At this point, each rank knows what it has to send to and what it is going to receive from the other ranks. + // The remaining action is about retrieving the additional graph information + // for the new geometrical quantities that will be sent by the neighbors + std::size_t const numMissingIndices = std::size( notPresentIndices_ ); + std::size_t const numGlobalMissingIndices = MpiWrapper::sum( numMissingIndices ); // A missing node can be missing on multiple ranks and that's OK. // Epetra_Map const missingIndicesMap( intConv< long long int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); Epetra_Map const missingIndicesMap( intConv< int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); // std::size_t const offset = MpiWrapper::prefixSum< std::size_t >( numMissingIndices ); - int const offset = *missingIndicesMap.MyGlobalElements(); + int const offset = *missingIndicesMap.MyGlobalElements(); // Needed to compute the global row // GEOS_LOG_RANK( "numMissingIndices, numGlobalMissingIndices, offset = " << numMissingIndices << ", " << numGlobalMissingIndices << ", " << offset ); + // The `missing` matrix is rectangular, + // the number of columns being the number of nodes in the mesh graph, + // the number of rows being the total number graph nodes that's missing on a rank. + // This matrix will Epetra_CrsMatrix missing( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); for( std::size_t i = 0; i < numMissingIndices; ++i ) { - missing.InsertGlobalValues( offset + i, 1, ones.data(), ¬PresentIndices[i] ); + missing.InsertGlobalValues( offset + i, 1, ones.data(), ¬PresentIndices_[i] ); } missing.FillComplete( ownedMap, missingIndicesMap ); @@ -697,7 +767,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o missingMappings.FillComplete( ownedMap, missingIndicesMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingMappings ); - MeshGraph ghosts; + MeshGraph ghosts; // TODO most likely this will be wrong. int ext = 0; std::vector< double > extValues( n ); @@ -715,7 +785,7 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o // GEOS_LOG( "ext 0 = " << std::size( s0 ) ); // GEOS_LOG( "ext 1 = " << std::size( s1 ) ); // GEOS_LOG_RANK( "ext = " << ext ); - int const index = notPresentIndices[i]; + int const index = notPresentIndices_[i]; switch( convert.getGeometricalType( index ) ) { case Geom::EDGE: @@ -756,26 +826,22 @@ std::tuple< MeshGraph, Ownerships > assembleAdjacencyMatrix( MeshGraph const & o } } - if( curRank == 2_mpi ) - { - GEOS_LOG_RANK( "neighboringConnexions = " << json( ghosts ) ); - std::map< MpiRank, std::set< CellGlbIdx > > tmp; - for( auto const & [geom, rk]: ownerships.cells ) - { - tmp[rk].insert( geom ); - } - for( auto const & [rk, geom]: tmp ) - { - GEOS_LOG_RANK( "ghost geom = " << rk << ": " << json( geom ) ); - } - } - GEOS_LOG_RANK( "my final neighbors are " << json( ownerships.neighbors ) ); - - return { std::move( ghosts ), std::move( ownerships ) }; - +// if( curRank == 2_mpi ) +// { +// GEOS_LOG_RANK( "neighboringConnexions = " << json( ghosts ) ); +// std::map< MpiRank, std::set< CellGlbIdx > > tmp; +// for( auto const & [geom, rk]: ownerships.cells ) +// { +// tmp[rk].insert( geom ); +// } +// for( auto const & [rk, geom]: tmp ) +// { +// GEOS_LOG_RANK( "ghost geom = " << rk << ": " << json( geom ) ); +// } +// } +// GEOS_LOG_RANK( "my final neighbors are " << json( ownerships.neighbors ) ); - // TODO to build the edges -> nodes map containing the all the ghost (i.e. the ghosted ones as well), - // Let's create a vector (or matrix?) full of ones where we have edges and multiply using the adjacency matrix. + return { std::move( ghosts ), std::move( recv ), std::move( send ) }; } /** @@ -806,17 +872,18 @@ buildGhostRankAndL2G( std::map< GI, MpiRank > const & m ) void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, - Ownerships const & ownerships ) + GhostRecv const & recv, + GhostSend const & send ) { - std::size_t const numNodes = std::size( ownerships.nodes ); - std::size_t const numEdges = std::size( ownerships.edges ); - std::size_t const numFaces = std::size( ownerships.faces ); - - auto [ghostRank, l2g] = buildGhostRankAndL2G( ownerships.edges ); - - NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); - EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); - FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( numFaces ) } ); +// std::size_t const numNodes = std::size( ownerships.nodes ); +// std::size_t const numEdges = std::size( ownerships.edges ); +// std::size_t const numFaces = std::size( ownerships.faces ); +// +// auto [ghostRank, l2g] = buildGhostRankAndL2G( ownerships.edges ); +// +// NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); +// EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); +// FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( numFaces ) } ); } std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, @@ -841,9 +908,9 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v // } // MpiWrapper::barrier(); - auto const [ghosts, ownerships] = assembleAdjacencyMatrix( owned, present, matrixOffsets, curRank ); + auto const [ghosts, recv, send] = assembleAdjacencyMatrix( owned, present, matrixOffsets, curRank ); - buildPods( owned, present, ghosts, ownerships ); + buildPods( owned, present, ghosts, recv, send ); return {}; } From 2736da6f61bac9d922030ae2a0c888561a95a0ef Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 23 May 2024 10:31:15 -0700 Subject: [PATCH 075/106] Use the exact size during the extraction + manage the node case for during the ordering. --- .../mesh/generators/NewGhosting.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index e73cdce710d..4ce57c994d3 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -661,11 +661,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph } int extracted = 0; - std::vector< double > extractedValues( n ); - std::vector< int > extractedIndices( n ); - ghostExchange.ExtractGlobalRowCopy( curRank.get(), int( n ), extracted, extractedValues.data(), extractedIndices.data() ); - extractedValues.resize( extracted ); - extractedIndices.resize( extracted ); + int const length = ghostExchange.NumGlobalEntries( curRank.get() ); + std::vector< double > extractedValues( length ); + std::vector< int > extractedIndices( length ); + ghostExchange.ExtractGlobalRowCopy( curRank.get(), length, extracted, extractedValues.data(), extractedIndices.data() ); + GEOS_ASSERT_EQ( extracted, length ); std::set< int > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); std::set< int > receivedIndices; // The indices my neighbors will send me. @@ -788,6 +788,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph int const index = notPresentIndices_[i]; switch( convert.getGeometricalType( index ) ) { + case Geom::NODE: + { + // Nothing to be done for nodes since they do not rely on any underlying information. + break; + } case Geom::EDGE: { GEOS_ASSERT_EQ( ext, 2 ); // TODO check that val != 0? From 30149120495f1f97bddfe5b9bf7cb5e337dccac6 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 24 May 2024 18:36:52 -0700 Subject: [PATCH 076/106] Conveys all the ordering, flip, shift information through the adjacency matrix. --- .../mesh/generators/NewGhosting.cpp | 358 +++++++++++++----- .../mesh/generators/NewGlobalNumbering.cpp | 21 +- .../mesh/generators/NewGlobalNumbering.hpp | 7 +- 3 files changed, 288 insertions(+), 98 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 4ce57c994d3..3f49a0d2d5b 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -115,10 +115,59 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, return result; } +/** + * @brief Complete set of information when a face refers to an edge. + * @details When a face refers to an edge, w.r.t. the canonical ordering of the edge, + * there's one node that comes before the other. + * The @c start parameter (which will be equal to either 0 or 1) will hold this information. + */ +struct EdgeInfo +{ + /// The global index of the edge the face is referring to. + EdgeGlbIdx index; + /// Which nodes comes first (and therefore which comes second) in the original edge numbering in the cell. + std::uint8_t start; // Where the initial index in the canonical face ordering was in the initial face. +}; + +void to_json( json & j, + const EdgeInfo & v ) // For display +{ + j = json{ { "index", v.index }, + { "start", v.start } }; +} + +/** + * @brief Complete set of information when a cell refers to a face. + * @details When a cell refers to a face, w.r.t. the canonical ordering of the face, + * the nodes of the face (in the cell) can be shifted, + * and travelled in a different direction (clockwise or counter-clockwise). + * The @c isFlipped parameter informs about the travel discrepancy. + * The @c start parameter informs about the shift. + * @note This class does not refer to how multiple faces are ordered by a cell, + * but to the precise information when refering to one given face. + */ +struct FaceInfo +{ + /// The global index of the face the cell is referring to. + FaceGlbIdx index; + /// Is the face travelled in the same direction as the canonical face. + bool isFlipped; + /// Where to start iterating over the nodes of the canonical faces to get back to the original face numbering in the cell. + std::uint8_t start; +}; + +void to_json( json & j, + const FaceInfo & v ) // For display +{ + j = json{ { "index", v.index }, + { "isFlipped", v.isFlipped }, + { "start", v.start } }; +} + struct MeshGraph { - std::map< CellGlbIdx, std::set< FaceGlbIdx > > c2f; // TODO What about the metadata (e.g. flip the face) - std::map< FaceGlbIdx, std::set< EdgeGlbIdx > > f2e; + std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; // TODO What about the metadata (e.g. flip the face) + std::map< FaceGlbIdx, std::vector< EdgeInfo > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? std::set< NodeGlbIdx > n; }; @@ -197,8 +246,8 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > NodeGlbIdx const & n0 = face[i], & n1 = face[i + 1]; std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); - f2e[fgi].insert( n2e.at( p1 ) ); - bool const flipped = p0 != p1; // TODO store somewhere. + EdgeInfo const info = { n2e.at( p1 ), std::uint8_t{ p0 == p1 } }; + f2e[fgi].emplace_back( info ); } ++fgi; } @@ -233,9 +282,10 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > vtkIdType const ngi = globalPtIds->GetValue( nli ); faceNodes[i] = NodeGlbIdx{ ngi }; } - std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes ); - owned.c2f[gci].insert( n2f.at( reorderedFaceNodes ) ); - // TODO... bool const flipped = ... compare nodes and reorderedNodes. Or ask `reorderFaceNodes` to tell + bool isFlipped; + std::uint8_t start; + std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes, isFlipped, start ); + owned.c2f[gci].emplace_back( FaceInfo{ n2f.at( reorderedFaceNodes ), isFlipped, start } ); } } @@ -427,6 +477,42 @@ class FindGeometricalType int const m_numEntries; }; +template< std::uint8_t N > +std::size_t encode( std::size_t const & basis, + std::array< std::size_t, N > const & array ) +{ + std::size_t result = 0; + for( auto i = 0; i < N; ++i ) + { + result *= basis; + result += array[i]; + } + return result; +} + +template< std::uint8_t N > +std::array< std::size_t, N > decode( std::size_t const & basis, + std::size_t const & input ) +{ + std::array< std::size_t, N > result, pows; + + for( std::size_t i = 0, j = N - 1, pow = 1; i < N; ++i, --j ) + { + pows[j] = pow; + pow *= basis; + } + + std::size_t dec = input; + for( auto i = 0; i < N; ++i ) + { + std::lldiv_t const div = std::lldiv( dec, pows[i] ); + result[i] = div.quot; + dec = div.rem; + } + return result; +} + + std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph const & owned, MeshGraph const & present, MaxGlbIdcs const & gis, @@ -450,9 +536,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph std::vector< int > ownedGlbIdcs, numEntriesPerRow; // TODO I couldn't use a vector of `std::size_t` std::vector< std::vector< int > > indices; + std::vector< std::vector< double > > values; ownedGlbIdcs.reserve( numOwned ); numEntriesPerRow.reserve( numOwned ); indices.reserve( numOwned ); + indices.reserve( numOwned ); std::vector< int > otherGlbIdcs; // TODO I couldn't use a vector of `std::size_t` otherGlbIdcs.reserve( numOther ); @@ -473,53 +561,89 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph for( NodeGlbIdx const & ngi: owned.n ) { - int const i = convert.fromNodeGlbIdx( ngi ); - ownedGlbIdcs.emplace_back( i ); - numEntriesPerRow.emplace_back( 1 ); - std::vector< int > const tmp( 1, ownedGlbIdcs.back() ); - indices.emplace_back( tmp ); + // Nodes depend on no other geometrical entity, + // so we only have one entry `1` in the diagonal of the matrix, + // because we add the identity to the adjacency matrix. + ownedGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); + numEntriesPerRow.emplace_back( 0 + 1 ); // `+1` comes from the diagonal + indices.emplace_back( 1, ownedGlbIdcs.back() ); + values.emplace_back( 1, ownedGlbIdcs.back() ); } for( auto const & [egi, nodes]: owned.e2n ) { - int const i = convert.fromEdgeGlbIdx( egi ); - ownedGlbIdcs.emplace_back( i ); - numEntriesPerRow.emplace_back( std::tuple_size_v< decltype( nodes ) > + 1 ); // `+1` comes from the identity - std::vector< int > const tmp{ int( std::get< 0 >( nodes ).get() ), int( std::get< 1 >( nodes ).get() ), ownedGlbIdcs.back() }; - indices.emplace_back( tmp ); + // Edges always depend on exactly 2 nodes, so this value can be hard coded. + // Also, edges have two different direction (starting from one point of the other). + // To keep the symmetry with the faces (see below), + // - we store the local index (0 or 1) to express this information. + // - we store the number of underlying nodes (2) on the diagonal. + size_t constexpr numNodes = std::tuple_size_v< decltype( nodes ) >; + + ownedGlbIdcs.emplace_back( convert.fromEdgeGlbIdx( egi ) ); + + numEntriesPerRow.emplace_back( numNodes + 1 ); // `+1` comes from the diagonal + indices.emplace_back( std::vector< int >{ convert.fromNodeGlbIdx( std::get< 0 >( nodes ) ), + convert.fromNodeGlbIdx( std::get< 1 >( nodes ) ), + ownedGlbIdcs.back() } ); + // Note that when storing a value that can be `0`, we always add `1`, + // (for edges but also later for faces and cells), + // to be sure that there will always be some a noticeable figure where we need one. + values.emplace_back( std::vector< double >{ 0 + 1., 1 + 1., numNodes } ); } for( auto const & [fgi, edges]: owned.f2e ) { - int const i = convert.fromFaceGlbIdx( fgi ); - ownedGlbIdcs.emplace_back( i ); - numEntriesPerRow.emplace_back( std::size( edges ) + 1 ); // `+1` comes from the identity - std::vector< int > tmp; - tmp.reserve( numEntriesPerRow.back() ); - for( EdgeGlbIdx const & egi: edges ) + // Faces point to multiple edges, but their edges can be flipped w.r.t. the canonical edges + // (ie minimal node global index comes first). + // In order not to lose this information (held by an `EdgeInfo` instance), we serialise + // - the `EdgeInfo` instance, + // - plus the order in which the edges are appearing, + // as an integer and store it as an entry in the matrix. + // On the diagonal, we'll store the number of edges of the face. + std::size_t const numEdges = std::size( edges ); + + ownedGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); + + numEntriesPerRow.emplace_back( numEdges + 1 ); // `+1` comes from the diagonal + std::vector< int > & ind = indices.emplace_back( numEntriesPerRow.back() ); + std::vector< double > & val = values.emplace_back( numEntriesPerRow.back() ); + for( std::size_t i = 0; i < numEdges; ++i ) { - tmp.emplace_back( convert.fromEdgeGlbIdx( egi ) ); + EdgeInfo const & edgeInfo = edges[i]; + ind[i] = convert.fromEdgeGlbIdx( edgeInfo.index ); + std::size_t const v = 1 + encode< 2 >( numEdges, { edgeInfo.start, i } ); + val[i] = double( v ); } - tmp.emplace_back( ownedGlbIdcs.back() ); - indices.emplace_back( tmp ); + ind.back() = ownedGlbIdcs.back(); + val.back() = double( numEdges ); } for( auto const & [cgi, faces]: owned.c2f ) { - int const i = convert.fromCellGlbIdx( cgi ); - ownedGlbIdcs.emplace_back( i ); - numEntriesPerRow.emplace_back( std::size( faces ) + 1 ); // `+1` comes from the identity - std::vector< int > tmp; - tmp.reserve( numEntriesPerRow.back() ); - for( FaceGlbIdx const & fgi: faces ) + // The same comment as for faces and edges applies for cells and faces (see above). + // The main differences being that + // - the `FaceInfo` as a little more information, + // - the diagonal stores the type of the cell which conveys the number of face, nodes, ordering... + std::size_t const numFaces = std::size( faces ); + + ownedGlbIdcs.emplace_back( convert.fromCellGlbIdx( cgi ) ); + + numEntriesPerRow.emplace_back( numFaces + 1 ); // `+1` comes from the diagonal + std::vector< int > & ind = indices.emplace_back( numEntriesPerRow.back() ); + std::vector< double > & val = values.emplace_back( numEntriesPerRow.back() ); + for( std::size_t i = 0; i < numFaces; ++i ) { - tmp.emplace_back( convert.fromFaceGlbIdx( fgi ) ); + FaceInfo const & faceInfo = faces[i]; + ind[i] = convert.fromFaceGlbIdx( faceInfo.index ); + std::size_t const v = 1 + encode< 3 >( numFaces, { faceInfo.isFlipped, faceInfo.start, i } ); + val[i] = double( v ); } - tmp.emplace_back( ownedGlbIdcs.back() ); - indices.emplace_back( tmp ); + ind.back() = ownedGlbIdcs.back(); + val.back() = double( numFaces ); // TODO This should be Hex and the not the number of faces... } std::sort( std::begin( ownedGlbIdcs ), std::end( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, std::size( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, std::size( numEntriesPerRow ) ); GEOS_ASSERT_EQ( numOwned, std::size( indices ) ); + GEOS_ASSERT_EQ( numOwned, std::size( values ) ); for( std::size_t i = 0; i < numOwned; ++i ) { GEOS_ASSERT_EQ( indices[i].size(), std::size_t( numEntriesPerRow[i] ) ); @@ -535,12 +659,12 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph for( std::size_t i = 0; i < numOwned; ++i ) { std::vector< int > const & rowIndices = indices[i]; - std::vector< double > const rowValues( std::size( rowIndices ), 1. ); + std::vector< double > const & rowValues = values[i]; GEOS_ASSERT_EQ( std::size( rowIndices ), std::size_t( numEntriesPerRow[i] ) ); + GEOS_ASSERT_EQ( std::size( rowValues ), std::size_t( numEntriesPerRow[i] ) ); upward.InsertGlobalValues( ownedGlbIdcs[i], std::size( rowIndices ), rowValues.data(), rowIndices.data() ); } -// upward.FillComplete(); upward.FillComplete( ownedMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/adj.mat", upward ); @@ -587,9 +711,12 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph // The `ghostingFootprint` matrix is rectangular, // the number of columns being the number of MPI ranks, // the number of rows being the number of nodes in the mesh graph. + // // For any MPI rank (ie column), a non-zero entry means // that the corresponding geometrical entry has to be eventually present onto the rank // for the ghosting to be effective. + // + // From `ghostingFootprint` we can extract where the current rank has to send any owned graph node. Epetra_CrsMatrix ghostingFootprint( multiply( commSize, indicator, upward ) ); ghostingFootprint.PutScalar( 1. ); // This can be done after the `FillComplete`. ghostingFootprint.FillComplete( mpiMap, ownedMap ); @@ -606,9 +733,20 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghostingFootprint.NumGlobalRows() ); } + // The `ghostExchange` matrix is rectangular, + // the number of columns being the number of nodes in the mesh graph, + // the number of rows being the number of MPI ranks. + // + // As the result of the multiplication of the `ghostingFootprint` matrix and the `ownership`, + // for each row owned (ie at the current MPI rank index), + // the value of the matrix term will provide the actual owning rank for all the . + // + // From `ghostExchange` we can extract which other rank will send to the current rank any graph node. Epetra_CrsMatrix ghostExchange( Epetra_DataAccess::Copy, mpiMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( ghostingFootprint, true, ownership, false, ghostExchange, false ); ghostExchange.FillComplete( ownedMap, mpiMap ); + // TODO Do I have to work with `ghostingFootprint` if I already have `ghostExchange` which may convey more information? + // TODO Maybe because of the ownership of the ranks: one is also the "scaled" transposed of the other. if( curRank == 0_mpi ) { @@ -618,7 +756,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostInfo.mat", ghostingFootprint ); int extracted2 = 0; - std::vector< double > extractedValues2( commSize ); + std::vector< double > extractedValues2( commSize ); // TODO improve with resize... std::vector< int > extractedIndices2( commSize ); GhostSend send; @@ -668,30 +806,17 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph GEOS_ASSERT_EQ( extracted, length ); std::set< int > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); - std::set< int > receivedIndices; // The indices my neighbors will send me. + std::set< int > receivedIndices; // The graph nodes that my neighbors will send me. std::set_difference( std::cbegin( allNeededIndices ), std::cend( allNeededIndices ), std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ), std::inserter( receivedIndices, std::end( receivedIndices ) ) ); - std::vector< int > notPresentIndices_; // TODO remove _ + std::vector< int > notPresentIndices; // The graphs nodes that are nor owned neither present by/on the current rank. std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), - std::back_inserter( notPresentIndices_ ) ); -// { -// std::set< int > const tmp( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); -// GEOS_LOG_RANK( "tmp = " << json( tmp ) ); -// GEOS_LOG_RANK( "receivedIndices = " << json( receivedIndices ) ); -// GEOS_ASSERT( tmp == receivedIndices ); -// } -// GEOS_LOG_RANK( "extracted = " << extracted ); -// if( curRank == 1_mpi ) -// { -// GEOS_LOG_RANK( "extractedIndices = " << json( extractedIndices ) ); -//// GEOS_LOG_RANK( "extracted values = " << json( extractedValues ) ); -// GEOS_LOG_RANK( "receivedIndices = " << json( receivedIndices ) ); -// } + std::back_inserter( notPresentIndices ) ); + GhostRecv recv; for( int i = 0; i < extracted; ++i ) -// for( int const & receivedIndex: receivedIndices ) { int const & index = extractedIndices[i]; if( receivedIndices.find( index ) == std::cend( receivedIndices ) ) // TODO make a map `receivedIndices -> mpi rank` @@ -733,8 +858,17 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph // At this point, each rank knows what it has to send to and what it is going to receive from the other ranks. // The remaining action is about retrieving the additional graph information - // for the new geometrical quantities that will be sent by the neighbors - std::size_t const numMissingIndices = std::size( notPresentIndices_ ); + // for the new geometrical quantities that will be sent by the neighbors. + // + // In order to do that, we build the `missing` matrix, which is rectangular, + // - the number of columns being the number of nodes in the mesh graph, + // - the number of rows being the total number graph nodes that's missing on ranks. + // - all its terms will be `1`. + // Combined with the adjacency matrix which conveya a lot of connections information, + // we'll be able to create the final `missingMappings`. + // We also create `maps` with specific MPI ranks ownerships and offsets, + // so we'll be able to retrieve the information on the ranks that need it (nothing more, nothing less). + std::size_t const numMissingIndices = std::size( notPresentIndices ); std::size_t const numGlobalMissingIndices = MpiWrapper::sum( numMissingIndices ); // A missing node can be missing on multiple ranks and that's OK. // Epetra_Map const missingIndicesMap( intConv< long long int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); @@ -744,23 +878,16 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph int const offset = *missingIndicesMap.MyGlobalElements(); // Needed to compute the global row // GEOS_LOG_RANK( "numMissingIndices, numGlobalMissingIndices, offset = " << numMissingIndices << ", " << numGlobalMissingIndices << ", " << offset ); - // The `missing` matrix is rectangular, - // the number of columns being the number of nodes in the mesh graph, - // the number of rows being the total number graph nodes that's missing on a rank. - // This matrix will Epetra_CrsMatrix missing( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); for( std::size_t i = 0; i < numMissingIndices; ++i ) { - missing.InsertGlobalValues( offset + i, 1, ones.data(), ¬PresentIndices_[i] ); + missing.InsertGlobalValues( offset + i, 1, ones.data(), ¬PresentIndices[i] ); } missing.FillComplete( ownedMap, missingIndicesMap ); - // TODO Don't put ones in downward: reassemble with the ordering (e.g. edges point to a first and last node) - auto tDownward = makeTranspose( upward ); // TODO give it to multiply! // EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-before.mat", *tDownward ); Epetra_Vector const zeros( ownedMap, true ); - tDownward->ReplaceDiagonalValues( zeros ); // TODO directly build zeros in the function call. // EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-after.mat", *tDownward ); Epetra_CrsMatrix missingMappings( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( missing, false, *tDownward, true, missingMappings, false ); @@ -770,22 +897,17 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph MeshGraph ghosts; // TODO most likely this will be wrong. int ext = 0; - std::vector< double > extValues( n ); - std::vector< int > extIndices( n ); -// double * extValues = nullptr; -// int * extIndices = nullptr; + std::vector< double > extValues; + std::vector< int > extIndices; for( int i = 0; i < int( numMissingIndices ); ++i ) { - missingMappings.ExtractGlobalRowCopy( offset + i, int( n ), ext, extValues.data(), extIndices.data() ); -// missingMappings.ExtractGlobalRowView( intConv< int >( offset + i ), ext, extValues ); -// missingMappings.ExtractGlobalRowView( intConv< int >( offset + i ), ext, extValues, extIndices ); -// Span< double > const s0( extValues, ext ); -// Span< int > const s1( extIndices, ext ); -// GEOS_LOG( "ext 0 = " << std::size( s0 ) ); -// GEOS_LOG( "ext 1 = " << std::size( s1 ) ); -// GEOS_LOG_RANK( "ext = " << ext ); - int const index = notPresentIndices_[i]; + int const length2 = missingMappings.NumGlobalEntries( offset + i ); + extValues.resize( length2 ); + extIndices.resize( length2 ); + missingMappings.ExtractGlobalRowCopy( offset + i, length2, ext, extValues.data(), extIndices.data() ); + GEOS_ASSERT_EQ( ext, length2 ); + int const index = notPresentIndices[i]; switch( convert.getGeometricalType( index ) ) { case Geom::NODE: @@ -795,33 +917,89 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph } case Geom::EDGE: { - GEOS_ASSERT_EQ( ext, 2 ); // TODO check that val != 0? + auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); + std::ptrdiff_t const numNodesIdx = std::distance( std::cbegin( extIndices ), cit ); + int const numNodes = int( extValues[numNodesIdx] ); + GEOS_ASSERT_EQ( ext, numNodes + 1 ); + GEOS_ASSERT_EQ( numNodes, 2 ); + std::array< NodeGlbIdx, 2 > order{}; + + for( int ii = 0; ii < ext; ++ii ) + { + if( ii == numNodesIdx ) + { + continue; + } + + NodeGlbIdx const ngi = convert.toNodeGlbIdx( extIndices[ii] ); + integer const ord = integer( extValues[ii] - 1 ); + GEOS_ASSERT( ord == 0 or ord == 1 ); + order[ord] = ngi; + } + GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numNodes ) ); EdgeGlbIdx const egi = convert.toEdgeGlbIdx( index ); - NodeGlbIdx const ngi0 = convert.toNodeGlbIdx( extIndices[0] ); - NodeGlbIdx const ngi1 = convert.toNodeGlbIdx( extIndices[1] ); - ghosts.e2n[egi] = std::minmax( { ngi0, ngi1 } ); + std::tuple< NodeGlbIdx, NodeGlbIdx > & tmp = ghosts.e2n[egi]; + std::get< 0 >( tmp ) = order[0]; + std::get< 1 >( tmp ) = order[1]; break; } case Geom::FACE: { - GEOS_ASSERT_EQ( ext, 4 ); // TODO temporary check for the dev - FaceGlbIdx const fgi = convert.toFaceGlbIdx( index ); - std::set< EdgeGlbIdx > & tmp = ghosts.f2e[fgi]; + auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); + std::ptrdiff_t const numEdgesIdx = std::distance( std::cbegin( extIndices ), cit ); + int const numEdges = int( extValues[numEdgesIdx] ); + GEOS_ASSERT_EQ( ext, numEdges + 1 ); + std::map< integer, EdgeInfo > order; for( int ii = 0; ii < ext; ++ii ) { - tmp.insert( convert.toEdgeGlbIdx( extIndices[ii] ) ); + if( ii == numEdgesIdx ) + { + continue; + } + + EdgeGlbIdx const egi = convert.toEdgeGlbIdx( extIndices[ii] ); + std::array< std::size_t, 2 > const decoded = decode< 2 >( numEdges, std::size_t( extValues[ii] - 1 ) ); + order[decoded[1]] = { egi, intConv< std::uint8_t >( decoded[0] ) }; + GEOS_ASSERT( decoded[0] == 0 or decoded[0] == 1 ); } + GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numEdges ) ); + FaceGlbIdx const fgi = convert.toFaceGlbIdx( index ); + std::vector< EdgeInfo > & tmp = ghosts.f2e[fgi]; + tmp.resize( numEdges ); + for( auto const & [ord, edgeInfo]: order ) + { + tmp[ord] = edgeInfo; + } +// GEOS_LOG_RANK( "faces ; index, extIndices, extValues, order = " << index << " | " << json( extIndices ) << " | " << json( extValues ) << " | " << json( order ) ); break; } case Geom::CELL: { - GEOS_ASSERT_EQ( ext, 6 ); // TODO temporary check for the dev - CellGlbIdx const cgi = convert.toCellGlbIdx( index ); - std::set< FaceGlbIdx > & tmp = ghosts.c2f[cgi]; + auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); + std::ptrdiff_t const numFacesIdx = std::distance( std::cbegin( extIndices ), cit ); + int const numFaces = int( extValues[numFacesIdx] ); // TODO This should receive the cell type instead. + GEOS_ASSERT_EQ( ext, numFaces + 1 ); + std::map< integer, FaceInfo > order; for( int ii = 0; ii < ext; ++ii ) { - tmp.insert( convert.toFaceGlbIdx( extIndices[ii] ) ); + if( ii == numFacesIdx ) + { + continue; + } + + FaceGlbIdx const fgi = convert.toFaceGlbIdx( extIndices[ii] ); + std::array< std::size_t, 3 > const decoded = decode< 3 >( numFaces, std::size_t( extValues[ii] - 1 ) ); + order[decoded[2]] = { fgi, intConv< bool >( decoded[0] ), intConv< std::uint8_t >( decoded[1] ) }; + } + GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numFaces ) ); + CellGlbIdx const cgi = convert.toCellGlbIdx( index ); + std::vector< FaceInfo > & tmp = ghosts.c2f[cgi]; + tmp.resize( numFaces ); + for( auto const & [ord, faceInfo]: order ) + { + tmp[ord] = faceInfo; } +// GEOS_LOG_RANK( "cells ; index, extIndices, extValues, order = " << index << " | " << json( extIndices ) << " | " << json( extValues ) << " | " << json( order ) ); break; } default: diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp index cef24c79837..aaee021b184 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -100,7 +100,7 @@ std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) return boundaryCellIdxs; } -Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) +Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes, bool & isFlipped, std::uint8_t & start ) // TODO unit test { std::size_t const n = nodes.size(); @@ -118,10 +118,17 @@ Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ) Face f; f.reserve( n ); - auto const it = std::min_element( nodes.cbegin(), nodes.cend() ); - std::size_t const minIdx = std::distance( nodes.cbegin(), it ); - int const increment = nodes[modulo( minIdx - 1 )] < nodes[modulo( minIdx + 1 )] ? -1 : 1; // TODO based on increment, I can say if the face is flipped or not. - integer i = minIdx; + auto const cit = std::min_element( std::cbegin( nodes ), std::cend( nodes ) ); + auto const minIdx = std::distance( std::cbegin( nodes ), cit ); + + std::size_t const prevIdx = modulo( intConv< integer >( minIdx - 1 ) ); + std::size_t const nextIdx = modulo( intConv< integer >( minIdx + 1 ) ); + int const increment = nodes[prevIdx] < nodes[nextIdx] ? -1 : 1; + + start = intConv< std::uint8_t >( minIdx ); + isFlipped = increment < 0; + + integer i = intConv< integer >( minIdx ); for( std::size_t count = 0; count < n; ++count, i = i + increment ) { f.emplace_back( nodes.at( modulo( i ) ) ); @@ -180,7 +187,9 @@ Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, vtkIdType const ngi = globalPtIds[nli]; nodes[i] = NodeGlbIdx{ ngi }; } - tmpFaces.emplace_back( reorderFaceNodes( nodes ) ); + bool isFlipped; + std::uint8_t start; + tmpFaces.emplace_back( reorderFaceNodes( nodes, isFlipped, start ) ); } } diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp index ae83b271830..118f5d4f45d 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp @@ -65,7 +65,9 @@ inline void from_json( const json & j, /** * @brief Order the nodes of the faces in a way that can be reproduced across the MPI ranks. - * @param nodes The list of nodes as provided by the mesh. + * @param[in] nodes The list of nodes as provided by the mesh. + * @param[out] isFlipped If the input faces (provided through the @c nodes) is flipped w.r.t. the canonical face. + * @param[out] start At which index we need to start in the canonical face to get back to the original face (provided through the @c nodes). * @return A face with the nodes in the appropriate order * @details The nodes will be ordered in the following way. * First, we look for the lowest node index. It will become the first node. @@ -80,7 +82,8 @@ inline void from_json( const json & j, * Except that edges having only two nodes, it's not necessary to implement a dedicated function * and std::minmax is enough. */ -Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes ); +Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes, bool & isFlipped, std::uint8_t & start ); + std::tuple< Buckets, BucketOffsets > doTheNewGlobalNumbering( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ); From ab38a0007aaf8d164a2411eed984bc6e6a0ee699 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 24 May 2024 18:43:57 -0700 Subject: [PATCH 077/106] Remove unused variable --- src/coreComponents/mesh/generators/NewGhosting.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 3f49a0d2d5b..839d6294afe 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -886,9 +886,6 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph missing.FillComplete( ownedMap, missingIndicesMap ); auto tDownward = makeTranspose( upward ); // TODO give it to multiply! -// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-before.mat", *tDownward ); - Epetra_Vector const zeros( ownedMap, true ); -// EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/tDownward-after.mat", *tDownward ); Epetra_CrsMatrix missingMappings( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); EpetraExt::MatrixMatrix::Multiply( missing, false, *tDownward, true, missingMappings, false ); missingMappings.FillComplete( ownedMap, missingIndicesMap ); From 25989c20a689ecf083a369b5e4df4a03fd41ec2f Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 30 May 2024 13:46:15 -0700 Subject: [PATCH 078/106] Small code refactoring --- .../mesh/generators/NewGhosting.cpp | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 839d6294afe..dbdd0f3ef35 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -718,7 +718,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph // // From `ghostingFootprint` we can extract where the current rank has to send any owned graph node. Epetra_CrsMatrix ghostingFootprint( multiply( commSize, indicator, upward ) ); - ghostingFootprint.PutScalar( 1. ); // This can be done after the `FillComplete`. + ghostingFootprint.PutScalar( 1. ); // This can be done after the `FillComplete`, but not with Tpetra! ghostingFootprint.FillComplete( mpiMap, ownedMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostingFootprint.mat", ghostingFootprint ); // We put `1` everywhere there's a non-zero entry, so we'll be able to compose with the `ownership` matrix. @@ -905,25 +905,29 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph missingMappings.ExtractGlobalRowCopy( offset + i, length2, ext, extValues.data(), extIndices.data() ); GEOS_ASSERT_EQ( ext, length2 ); int const index = notPresentIndices[i]; - switch( convert.getGeometricalType( index ) ) + Geom const geometricalType = convert.getGeometricalType( index ); + if ( geometricalType == Geom::NODE ) + { + // Nothing to be done for nodes since they do not rely on any underlying information. + continue; + } + + auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); + std::ptrdiff_t const numGeomQuantitiesIdx = std::distance( std::cbegin( extIndices ), cit ); + int const numGeomQuantities = int( extValues[numGeomQuantitiesIdx] ); + GEOS_ASSERT_EQ( ext, numGeomQuantities + 1 ); + + switch( geometricalType ) { - case Geom::NODE: - { - // Nothing to be done for nodes since they do not rely on any underlying information. - break; - } case Geom::EDGE: { - auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); - std::ptrdiff_t const numNodesIdx = std::distance( std::cbegin( extIndices ), cit ); - int const numNodes = int( extValues[numNodesIdx] ); - GEOS_ASSERT_EQ( ext, numNodes + 1 ); + int const & numNodes = numGeomQuantities; // Alias GEOS_ASSERT_EQ( numNodes, 2 ); std::array< NodeGlbIdx, 2 > order{}; for( int ii = 0; ii < ext; ++ii ) { - if( ii == numNodesIdx ) + if( ii == numGeomQuantitiesIdx ) { continue; } @@ -942,14 +946,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph } case Geom::FACE: { - auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); - std::ptrdiff_t const numEdgesIdx = std::distance( std::cbegin( extIndices ), cit ); - int const numEdges = int( extValues[numEdgesIdx] ); - GEOS_ASSERT_EQ( ext, numEdges + 1 ); + int const & numEdges = numGeomQuantities; // Alias std::map< integer, EdgeInfo > order; for( int ii = 0; ii < ext; ++ii ) { - if( ii == numEdgesIdx ) + if( ii == numGeomQuantitiesIdx ) { continue; } @@ -972,14 +973,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph } case Geom::CELL: { - auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); - std::ptrdiff_t const numFacesIdx = std::distance( std::cbegin( extIndices ), cit ); - int const numFaces = int( extValues[numFacesIdx] ); // TODO This should receive the cell type instead. - GEOS_ASSERT_EQ( ext, numFaces + 1 ); + int const & numFaces = numGeomQuantities; // Alias // TODO This should receive the cell type instead. std::map< integer, FaceInfo > order; for( int ii = 0; ii < ext; ++ii ) { - if( ii == numFacesIdx ) + if( ii == numGeomQuantitiesIdx ) { continue; } From 68cd212f68f80b3609e8fd273e06d33b882b7825 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 30 May 2024 14:08:07 -0700 Subject: [PATCH 079/106] Move buidPods to its own file + some structure in headers for definitions. --- src/coreComponents/mesh/CMakeLists.txt | 2 + .../mesh/generators/BuildPods.cpp | 38 ++++++++ .../mesh/generators/BuildPods.hpp | 92 +++++++++++++++++++ .../mesh/generators/NewGhosting.cpp | 77 +--------------- 4 files changed, 134 insertions(+), 75 deletions(-) create mode 100644 src/coreComponents/mesh/generators/BuildPods.cpp create mode 100644 src/coreComponents/mesh/generators/BuildPods.hpp diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index 2fa54d84087..eb19c913718 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -182,6 +182,7 @@ list( APPEND dependencyList NamedType nlohmann_json::nlohmann_json trilinos ) if( ENABLE_VTK ) message(STATUS "Adding VTK readers") set( mesh_headers ${mesh_headers} + generators/BuildPods.hpp generators/CollocatedNodes.hpp generators/NewGhosting.hpp generators/NewGlobalNumbering.hpp @@ -192,6 +193,7 @@ if( ENABLE_VTK ) generators/VTKUtilities.hpp ) set( mesh_sources ${mesh_sources} + generators/BuildPods.cpp generators/CollocatedNodes.cpp generators/NewGhosting.cpp generators/NewGlobalNumbering.cpp diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp new file mode 100644 index 00000000000..0c1ae64156f --- /dev/null +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -0,0 +1,38 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "BuildPods.hpp" + + +namespace geos::ghosting +{ + +void buildPods( MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts, + GhostRecv const & recv, + GhostSend const & send ) +{ +// std::size_t const numNodes = std::size( ownerships.nodes ); +// std::size_t const numEdges = std::size( ownerships.edges ); +// std::size_t const numFaces = std::size( ownerships.faces ); +// +// auto [ghostRank, l2g] = buildGhostRankAndL2G( ownerships.edges ); +// +// NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); +// EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); +// FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( numFaces ) } ); +} + +} diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp new file mode 100644 index 00000000000..15279daa325 --- /dev/null +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -0,0 +1,92 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_BUILDPODS_HPP +#define GEOS_BUILDPODS_HPP + +#include "Indices.hpp" + +#include +#include + +namespace geos::ghosting +{ + +/** + * @brief Complete set of information when a cell refers to a face. + * @details When a cell refers to a face, w.r.t. the canonical ordering of the face, + * the nodes of the face (in the cell) can be shifted, + * and travelled in a different direction (clockwise or counter-clockwise). + * The @c isFlipped parameter informs about the travel discrepancy. + * The @c start parameter informs about the shift. + * @note This class does not refer to how multiple faces are ordered by a cell, + * but to the precise information when refering to one given face. + */ +struct FaceInfo +{ + /// The global index of the face the cell is referring to. + FaceGlbIdx index; + /// Is the face travelled in the same direction as the canonical face. + bool isFlipped; + /// Where to start iterating over the nodes of the canonical faces to get back to the original face numbering in the cell. + std::uint8_t start; +}; + +/** + * @brief Complete set of information when a face refers to an edge. + * @details When a face refers to an edge, w.r.t. the canonical ordering of the edge, + * there's one node that comes before the other. + * The @c start parameter (which will be equal to either 0 or 1) will hold this information. + */ +struct EdgeInfo +{ + /// The global index of the edge the face is referring to. + EdgeGlbIdx index; + /// Which nodes comes first (and therefore which comes second) in the original edge numbering in the cell. + std::uint8_t start; // Where the initial index in the canonical face ordering was in the initial face. +}; + +struct MeshGraph +{ + std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; // TODO What about the metadata (e.g. flip the face) + std::map< FaceGlbIdx, std::vector< EdgeInfo > > f2e; + std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? + std::set< NodeGlbIdx > n; +}; + +struct GhostSend +{ + std::map< NodeGlbIdx, std::set< MpiRank > > nodes; + std::map< EdgeGlbIdx, std::set< MpiRank > > edges; + std::map< FaceGlbIdx, std::set< MpiRank > > faces; + std::map< CellGlbIdx, std::set< MpiRank > > cells; +}; + +struct GhostRecv +{ + std::map< NodeGlbIdx, MpiRank > nodes; + std::map< EdgeGlbIdx, MpiRank > edges; + std::map< FaceGlbIdx, MpiRank > faces; + std::map< CellGlbIdx, MpiRank > cells; +}; + +void buildPods( MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts, + GhostRecv const & recv, + GhostSend const & send ); + +} + +#endif //GEOS_BUILDPODS_HPP diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index dbdd0f3ef35..54a356a1382 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -16,6 +16,8 @@ #include "NewGlobalNumbering.hpp" +#include "BuildPods.hpp" + #include "Pods.hpp" #include "common/MpiWrapper.hpp" @@ -115,20 +117,6 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, return result; } -/** - * @brief Complete set of information when a face refers to an edge. - * @details When a face refers to an edge, w.r.t. the canonical ordering of the edge, - * there's one node that comes before the other. - * The @c start parameter (which will be equal to either 0 or 1) will hold this information. - */ -struct EdgeInfo -{ - /// The global index of the edge the face is referring to. - EdgeGlbIdx index; - /// Which nodes comes first (and therefore which comes second) in the original edge numbering in the cell. - std::uint8_t start; // Where the initial index in the canonical face ordering was in the initial face. -}; - void to_json( json & j, const EdgeInfo & v ) // For display { @@ -136,26 +124,6 @@ void to_json( json & j, { "start", v.start } }; } -/** - * @brief Complete set of information when a cell refers to a face. - * @details When a cell refers to a face, w.r.t. the canonical ordering of the face, - * the nodes of the face (in the cell) can be shifted, - * and travelled in a different direction (clockwise or counter-clockwise). - * The @c isFlipped parameter informs about the travel discrepancy. - * The @c start parameter informs about the shift. - * @note This class does not refer to how multiple faces are ordered by a cell, - * but to the precise information when refering to one given face. - */ -struct FaceInfo -{ - /// The global index of the face the cell is referring to. - FaceGlbIdx index; - /// Is the face travelled in the same direction as the canonical face. - bool isFlipped; - /// Where to start iterating over the nodes of the canonical faces to get back to the original face numbering in the cell. - std::uint8_t start; -}; - void to_json( json & j, const FaceInfo & v ) // For display { @@ -164,14 +132,6 @@ void to_json( json & j, { "start", v.start } }; } -struct MeshGraph -{ - std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; // TODO What about the metadata (e.g. flip the face) - std::map< FaceGlbIdx, std::vector< EdgeInfo > > f2e; - std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? - std::set< NodeGlbIdx > n; -}; - void to_json( json & j, const MeshGraph & v ) // For display { @@ -304,14 +264,6 @@ std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, return ptr; } -struct GhostSend -{ - std::map< NodeGlbIdx, std::set< MpiRank > > nodes; - std::map< EdgeGlbIdx, std::set< MpiRank > > edges; - std::map< FaceGlbIdx, std::set< MpiRank > > faces; - std::map< CellGlbIdx, std::set< MpiRank > > cells; -}; - void to_json( json & j, const GhostSend & v ) { @@ -321,14 +273,6 @@ void to_json( json & j, { "cells", v.cells } }; } -struct GhostRecv -{ - std::map< NodeGlbIdx, MpiRank > nodes; - std::map< EdgeGlbIdx, MpiRank > edges; - std::map< FaceGlbIdx, MpiRank > faces; - std::map< CellGlbIdx, MpiRank > cells; -}; - void to_json( json & j, const GhostRecv & v ) { @@ -1047,23 +991,6 @@ buildGhostRankAndL2G( std::map< GI, MpiRank > const & m ) return std::make_tuple( ghostRank, l2g ); } -void buildPods( MeshGraph const & owned, - MeshGraph const & present, - MeshGraph const & ghosts, - GhostRecv const & recv, - GhostSend const & send ) -{ -// std::size_t const numNodes = std::size( ownerships.nodes ); -// std::size_t const numEdges = std::size( ownerships.edges ); -// std::size_t const numFaces = std::size( ownerships.faces ); -// -// auto [ghostRank, l2g] = buildGhostRankAndL2G( ownerships.edges ); -// -// NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); -// EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); -// FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( numFaces ) } ); -} - std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { From d62326f9a92249ec5a4274878907508d5fac72be Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 31 May 2024 12:56:44 -0700 Subject: [PATCH 080/106] Building the EdgeMgrImpl --- .../mesh/generators/BuildPods.cpp | 197 +++++++++++++++++- .../mesh/generators/Indices.hpp | 31 ++- .../mesh/generators/NewGhosting.cpp | 34 +-- src/coreComponents/mesh/generators/Pods.cpp | 34 ++- src/coreComponents/mesh/generators/Pods.hpp | 18 +- 5 files changed, 248 insertions(+), 66 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 0c1ae64156f..6f100d30993 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -14,25 +14,204 @@ #include "BuildPods.hpp" +#include "Pods.hpp" + +//#include "common/MpiWrapper.hpp" +#include "codingUtilities/Utilities.hpp" + +#include "LvArray/src/ArrayOfArraysView.hpp" namespace geos::ghosting { +struct GlobalNumberings +{ + std::map< NodeGlbIdx, NodeLocIdx > ng2l; + std::map< NodeLocIdx, NodeGlbIdx > nl2g; + + std::map< EdgeGlbIdx, EdgeLocIdx > eg2l; + std::map< EdgeLocIdx, EdgeGlbIdx > el2g; // TODO Do we want to make this a vector already? It's surely the most efficient way to build it. + + std::map< FaceGlbIdx, FaceLocIdx > fg2l; + std::map< FaceLocIdx, FaceGlbIdx > fl2g; + + std::map< CellGlbIdx, CellLocIdx > cg2l; + std::map< CellLocIdx, CellGlbIdx > cl2g; +}; + + +template< class GI, class LI > +void buildL2GMappings( std::set< GI > const & owned, + std::set< GI > const & present, + std::set< GI > const & ghosts, + std::map< GI, LI > & g2l, + std::map< LI, GI > & l2g ) // TODO we can make it a vector -> simple push_backs will be enough to build it +{ + g2l.clear(); + l2g.clear(); + + LI li{ 0 }; + for( std::set< GI > const & gis: { owned, present, ghosts } ) + { + for( GI const & gi: gis ) + { + g2l.insert( { gi, li } ); + l2g.insert( { li, gi } ); + ++li; + } + } + + // TODO add a check to see if we have the full range of local indices. +} + +GlobalNumberings buildL2GMappings( MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts ) +{ + GlobalNumberings result; + + // Use `std::ranges::views::keys` when switching to C++20 + auto const keys = []( auto const & m ) + { + return mapKeys< std::set >( m ); + }; + + buildL2GMappings( owned.n, present.n, ghosts.n, result.ng2l, result.nl2g ); + buildL2GMappings( keys( owned.e2n ), keys( present.e2n ), keys( ghosts.e2n ), result.eg2l, result.el2g ); + buildL2GMappings( keys( owned.f2e ), keys( present.f2e ), keys( ghosts.f2e ), result.fg2l, result.fl2g ); + buildL2GMappings( keys( owned.c2f ), keys( present.c2f ), keys( ghosts.c2f ), result.cg2l, result.cl2g ); + + return result; +} + +template< class GI, class LI > +std::vector< integer > buildGhostRank( std::map< GI, LI > const & g2l, + std::map< GI, std::set< MpiRank > > const & send, + std::map< GI, MpiRank > const & recv ) +{ + std::vector< integer > ghostRank( std::size( g2l ), -2 ); + + for( auto const & [gi, _]: send ) + { + ghostRank[g2l.at( gi ).get()] = -1; + } + for( auto const & [gi, rank]: recv ) + { + ghostRank[g2l.at( gi ).get()] = rank.get(); + } + + return ghostRank; +} + +EdgeMgrImpl makeFlavorLessEdgeMgrImpl( std::size_t const & numEdges, + std::vector< integer > const & ghostRank, + std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, + std::map< EdgeLocIdx, std::vector< FaceLocIdx > > const & e2f, + std::map< EdgeGlbIdx, EdgeLocIdx > const & eg2l, + std::map< EdgeLocIdx, EdgeGlbIdx > const & el2g ) +{ + array2d< localIndex > e2n_( numEdges, 2 ); + for( int i = 0; i < intConv< int >( numEdges ); ++i ) + { + std::tuple< NodeLocIdx, NodeLocIdx > nlis = e2n.at( EdgeLocIdx{ i } ); + e2n_[i][0] = std::get< 0 >( nlis ).get(); + e2n_[i][1] = std::get< 1 >( nlis ).get(); + } + + ArrayOfArrays< localIndex > e2f_; + std::vector< int > sizes; + sizes.reserve( numEdges ); + for( auto const & [_, flis]: e2f ) + { + sizes.emplace_back( std::size( flis ) ); + } + GEOS_ASSERT_EQ( std::size( sizes ), numEdges ); + e2f_.resizeFromCapacities< serialPolicy >( numEdges, sizes.data() ); + for( auto const & [eli, flis]: e2f ) + { + for( FaceLocIdx const & fli: flis ) + { + e2f_.emplaceBack( eli.get(), fli.get() ); + } + } + + array1d< globalIndex > l2g( numEdges ); + unordered_map< globalIndex, localIndex > g2l; + for( auto const & [egi, eli]: eg2l ) + { + l2g[eli.get()] = egi.get(); + g2l[egi.get()] = eli.get(); + } + + array1d< integer > ghostRank_( numEdges ); + for( integer const & gr: ghostRank ) + { + ghostRank_.emplace_back( gr ); + } + + return EdgeMgrImpl( numEdges, std::move( ghostRank_ ), std::move( e2n_ ), std::move( e2f_ ), std::move( g2l ), std::move( l2g ) ); +} + +EdgeMgrImpl buildEdgeMgr( GlobalNumberings const & numberings, + MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts, + GhostRecv const & recv, + GhostSend const & send ) +{ + // Total number of edges available in the rank (including the ghosted edges). + std::size_t const numEdges = std::size( numberings.el2g ); + + // Building the ghost rank. + std::vector< integer > const ghostRank = buildGhostRank( numberings.eg2l, send.edges, recv.edges ); + + // Building the edges to nodes mapping + std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > e2n; + for( MeshGraph const & graph: { owned, present, ghosts } ) + { + for( auto const & [egi, ngis]: graph.e2n ) + { + NodeLocIdx const nli0 = numberings.ng2l.at( std::get< 0 >( ngis ) ); + NodeLocIdx const nli1 = numberings.ng2l.at( std::get< 1 >( ngis ) ); + e2n[numberings.eg2l.at( egi )] = { nli0, nli1 }; + } + } + + // Building the edges to nodes mapping + std::map< EdgeLocIdx, std::vector< FaceLocIdx > > e2f; + for( MeshGraph const & graph: { owned, present, ghosts } ) + { + for( auto const & [fgi, edgeInfos]: graph.f2e ) + { + for( EdgeInfo const & edgeInfo: edgeInfos ) + { + e2f[numberings.eg2l.at( edgeInfo.index )].emplace_back( numberings.fg2l.at( fgi ) ); + } + } + } + + return makeFlavorLessEdgeMgrImpl( numEdges, ghostRank, e2n, e2f, numberings.eg2l, numberings.el2g ); +} + void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, GhostRecv const & recv, GhostSend const & send ) { -// std::size_t const numNodes = std::size( ownerships.nodes ); -// std::size_t const numEdges = std::size( ownerships.edges ); -// std::size_t const numFaces = std::size( ownerships.faces ); -// -// auto [ghostRank, l2g] = buildGhostRankAndL2G( ownerships.edges ); -// -// NodeMgrImpl const nodeMgr( NodeLocIdx{ intConv< localIndex >( numNodes ) } ); -// EdgeMgrImpl const edgeMgr( EdgeLocIdx{ intConv< localIndex >( numEdges ) }, std::move( ghostRank ), std::move( l2g ) ); -// FaceMgrImpl const faceMgr( FaceLocIdx{ intConv< localIndex >( numFaces ) } ); + GlobalNumberings const numberings = buildL2GMappings( owned, present, ghosts ); +// GEOS_LOG_RANK( "numberings ng2l = " << json( numberings.ng2l ) ); +// GEOS_LOG_RANK( "owned mesh graph nodes = " << json( owned.n ) ); +// GEOS_LOG_RANK( "present mesh graph nodes = " << json( present.n ) ); +// GEOS_LOG_RANK( "ghosts mesh graph nodes = " << json( ghosts.n ) ); +// GEOS_LOG_RANK( "owned mesh graph e2n = " << json( owned.e2n ) ); +// GEOS_LOG_RANK( "present mesh graph e2n = " << json( present.e2n ) ); +// GEOS_LOG_RANK( "ghosts mesh graph e2n = " << json( ghosts.e2n ) ); +// auto const EdgeMgr = buildEdgeMgr( owned, present, ghosts, recv, send ); + +// MpiWrapper::barrier(); + + buildEdgeMgr( numberings, owned, present, ghosts, recv, send ); } } diff --git a/src/coreComponents/mesh/generators/Indices.hpp b/src/coreComponents/mesh/generators/Indices.hpp index 9c1e9765993..62235a3129e 100644 --- a/src/coreComponents/mesh/generators/Indices.hpp +++ b/src/coreComponents/mesh/generators/Indices.hpp @@ -37,13 +37,13 @@ OUTPUT intConv( INPUT input ) return LvArray::integerConversion< OUTPUT >( input ); } -using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable >; +using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable >; +using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; -using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable >; +using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; -using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable >; +using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; @@ -117,11 +117,22 @@ inline void to_json( json & j, j = v.get(); } +inline void to_json( json & j, + const NodeLocIdx & v ) +{ + j = v.get(); +} + inline void to_json( json & j, const EdgeGlbIdx & v ) { j = v.get(); } +inline void to_json( json & j, + const EdgeLocIdx & v ) +{ + j = v.get(); +} inline void from_json( const json & j, EdgeGlbIdx & v ) @@ -135,6 +146,12 @@ inline void to_json( json & j, j = v.get(); } +inline void to_json( json & j, + const FaceLocIdx & v ) +{ + j = v.get(); +} + inline void from_json( const json & j, FaceGlbIdx & v ) { @@ -147,6 +164,12 @@ inline void to_json( json & j, j = v.get(); } +inline void to_json( json & j, + const CellLocIdx & v ) +{ + j = v.get(); +} + } #endif //GEOS_INDICES_HPP diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 54a356a1382..22720e3dd96 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -850,9 +850,14 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph GEOS_ASSERT_EQ( ext, length2 ); int const index = notPresentIndices[i]; Geom const geometricalType = convert.getGeometricalType( index ); - if ( geometricalType == Geom::NODE ) + if( geometricalType == Geom::NODE ) { - // Nothing to be done for nodes since they do not rely on any underlying information. + // The case of nodes is a bit different from the other cases + // because nodes do not rely on other geometrical quantities. + // We simply have to extract and store their own index. + GEOS_ASSERT_EQ( length2, 1 ); + GEOS_ASSERT_EQ( extIndices[0], int(extValues[0]) ); + ghosts.n.insert( convert.toNodeGlbIdx( extIndices[0] ) ); continue; } @@ -966,31 +971,6 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph return { std::move( ghosts ), std::move( recv ), std::move( send ) }; } -/** - * @brief - * @tparam GI A global index type - * @param m - * @return - */ -template< class GI > -std::tuple< std::vector< MpiRank >, std::vector< GI > > -buildGhostRankAndL2G( std::map< GI, MpiRank > const & m ) -{ - std::size_t const size = std::size( m ); - - std::vector< MpiRank > ghostRank; - ghostRank.reserve( size ); - std::vector< GI > l2g; - l2g.reserve( size ); - for( auto const & [t, rank]: m ) - { - ghostRank.emplace_back( rank ); - l2g.emplace_back( t ); - } - - return std::make_tuple( ghostRank, l2g ); -} - std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ) { diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 61e434f8475..3552b87cd85 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -56,49 +56,43 @@ std::map< string, SortedArray< localIndex > > const & NodeMgrImpl::getNodeSets() return m_todo; } -EdgeMgrImpl::EdgeMgrImpl( EdgeLocIdx const & numEdges, - std::vector< MpiRank > && ghostRank, - std::vector< EdgeGlbIdx > && l2g ) +EdgeMgrImpl::EdgeMgrImpl( std::size_t numEdges, + array1d< integer > && ghostRank, + array2d< localIndex > && e2n, + ArrayOfArrays< localIndex > && e2f, + unordered_map< globalIndex, localIndex > && g2l, + array1d< globalIndex > && l2g ) : m_numEdges( numEdges ), m_ghostRank( ghostRank ), + m_e2n( e2n ), + m_e2f( e2f ), + m_g2l( g2l ), m_l2g( l2g ) { } localIndex EdgeMgrImpl::numEdges() const { - return intConv< localIndex >( m_numEdges.get() ); + return m_numEdges; } array2d< localIndex > EdgeMgrImpl::getEdgeToNodes() const { - return {}; + return m_e2n; } ArrayOfArrays< localIndex > EdgeMgrImpl::getEdgeToFaces() const { - return {}; + return m_e2f; } array1d< integer > EdgeMgrImpl::getGhostRank() const { - array1d< integer > result; - result.reserve( std::size( m_ghostRank ) ); - for( MpiRank const & rank: m_ghostRank ) - { - result.emplace_back( rank.get() ); - } - return result; + return m_ghostRank; } array1d< globalIndex > EdgeMgrImpl::getLocalToGlobal() const { - array1d< globalIndex > result; - result.reserve( std::size( m_l2g ) ); - for( EdgeGlbIdx const & edge: m_l2g ) - { - result.emplace_back( edge.get() ); - } - return result; + return m_l2g; } FaceMgrImpl::FaceMgrImpl( FaceLocIdx const & numFaces ) diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index da867955606..7562ae8a5cc 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -52,9 +52,12 @@ class NodeMgrImpl : public generators::NodeMgr class EdgeMgrImpl : public generators::EdgeMgr { public: - EdgeMgrImpl( EdgeLocIdx const & numEdges, - std::vector< MpiRank > && ghostRank, - std::vector< EdgeGlbIdx > && l2g ); + EdgeMgrImpl( std::size_t numEdges, + array1d< integer > && ghostRank, + array2d< localIndex > && e2n, + ArrayOfArrays< localIndex > && e2f, + unordered_map< globalIndex, localIndex > && g2l, + array1d< globalIndex > && l2g ); [[nodiscard]] localIndex numEdges() const override; @@ -67,9 +70,12 @@ class EdgeMgrImpl : public generators::EdgeMgr [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; private: - EdgeLocIdx m_numEdges; - std::vector< MpiRank > m_ghostRank; - std::vector< EdgeGlbIdx > m_l2g; + localIndex m_numEdges; + array1d< integer > m_ghostRank; + array2d< localIndex > m_e2n; + ArrayOfArrays< localIndex > m_e2f; + unordered_map< globalIndex, localIndex > m_g2l; + array1d< globalIndex > m_l2g; }; class FaceMgrImpl : public generators::FaceMgr From cd30c9c97972ed6a17ca15b595f3bf60023ed7d7 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:05:23 -0700 Subject: [PATCH 081/106] Merge mesh graphs into a big one. --- .../mesh/generators/BuildPods.cpp | 112 ++++++++++++------ .../mesh/generators/BuildPods.hpp | 2 +- .../mesh/generators/NewGhosting.cpp | 2 +- 3 files changed, 77 insertions(+), 39 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 6f100d30993..19db620ddcf 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -41,9 +41,7 @@ struct GlobalNumberings template< class GI, class LI > -void buildL2GMappings( std::set< GI > const & owned, - std::set< GI > const & present, - std::set< GI > const & ghosts, +void buildL2GMappings( std::set< GI > const & gis, std::map< GI, LI > & g2l, std::map< LI, GI > & l2g ) // TODO we can make it a vector -> simple push_backs will be enough to build it { @@ -51,22 +49,17 @@ void buildL2GMappings( std::set< GI > const & owned, l2g.clear(); LI li{ 0 }; - for( std::set< GI > const & gis: { owned, present, ghosts } ) + for( GI const & gi: gis ) { - for( GI const & gi: gis ) - { - g2l.insert( { gi, li } ); - l2g.insert( { li, gi } ); - ++li; - } + g2l.insert( { gi, li } ); + l2g.insert( { li, gi } ); + ++li; } // TODO add a check to see if we have the full range of local indices. } -GlobalNumberings buildL2GMappings( MeshGraph const & owned, - MeshGraph const & present, - MeshGraph const & ghosts ) +GlobalNumberings buildL2GMappings( MeshGraph const & graph ) { GlobalNumberings result; @@ -76,10 +69,10 @@ GlobalNumberings buildL2GMappings( MeshGraph const & owned, return mapKeys< std::set >( m ); }; - buildL2GMappings( owned.n, present.n, ghosts.n, result.ng2l, result.nl2g ); - buildL2GMappings( keys( owned.e2n ), keys( present.e2n ), keys( ghosts.e2n ), result.eg2l, result.el2g ); - buildL2GMappings( keys( owned.f2e ), keys( present.f2e ), keys( ghosts.f2e ), result.fg2l, result.fl2g ); - buildL2GMappings( keys( owned.c2f ), keys( present.c2f ), keys( ghosts.c2f ), result.cg2l, result.cl2g ); + buildL2GMappings( graph.n, result.ng2l, result.nl2g ); + buildL2GMappings( keys( graph.e2n ), result.eg2l, result.el2g ); + buildL2GMappings( keys( graph.f2e ), result.fg2l, result.fl2g ); + buildL2GMappings( keys( graph.c2f ), result.cg2l, result.cl2g ); return result; } @@ -103,7 +96,7 @@ std::vector< integer > buildGhostRank( std::map< GI, LI > const & g2l, return ghostRank; } -EdgeMgrImpl makeFlavorLessEdgeMgrImpl( std::size_t const & numEdges, +EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::vector< integer > const & ghostRank, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, std::map< EdgeLocIdx, std::vector< FaceLocIdx > > const & e2f, @@ -153,9 +146,7 @@ EdgeMgrImpl makeFlavorLessEdgeMgrImpl( std::size_t const & numEdges, } EdgeMgrImpl buildEdgeMgr( GlobalNumberings const & numberings, - MeshGraph const & owned, - MeshGraph const & present, - MeshGraph const & ghosts, + MeshGraph const & graph, GhostRecv const & recv, GhostSend const & send ) { @@ -167,30 +158,74 @@ EdgeMgrImpl buildEdgeMgr( GlobalNumberings const & numberings, // Building the edges to nodes mapping std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > e2n; - for( MeshGraph const & graph: { owned, present, ghosts } ) + for( auto const & [egi, ngis]: graph.e2n ) { - for( auto const & [egi, ngis]: graph.e2n ) - { - NodeLocIdx const nli0 = numberings.ng2l.at( std::get< 0 >( ngis ) ); - NodeLocIdx const nli1 = numberings.ng2l.at( std::get< 1 >( ngis ) ); - e2n[numberings.eg2l.at( egi )] = { nli0, nli1 }; - } + NodeLocIdx const nli0 = numberings.ng2l.at( std::get< 0 >( ngis ) ); + NodeLocIdx const nli1 = numberings.ng2l.at( std::get< 1 >( ngis ) ); + e2n[numberings.eg2l.at( egi )] = { nli0, nli1 }; } // Building the edges to nodes mapping std::map< EdgeLocIdx, std::vector< FaceLocIdx > > e2f; - for( MeshGraph const & graph: { owned, present, ghosts } ) + for( auto const & [fgi, edgeInfos]: graph.f2e ) + { + for( EdgeInfo const & edgeInfo: edgeInfos ) + { + e2f[numberings.eg2l.at( edgeInfo.index )].emplace_back( numberings.fg2l.at( fgi ) ); + } + } + + return makeFlavorlessEdgeMgrImpl( numEdges, ghostRank, e2n, e2f, numberings.eg2l, numberings.el2g ); +} + +void buildFaceMgr( GlobalNumberings const & numberings, + MeshGraph const & graph, + GhostRecv const & recv, + GhostSend const & send ) +{ + // Total number of faces available in the rank (including the ghosted edges). + std::size_t const numFaces = std::size( numberings.fl2g ); + + // Building the ghost rank. + std::vector< integer > const ghostRank = buildGhostRank( numberings.fg2l, send.faces, recv.faces ); + + // Building the `f2n` and `f2e` mappings + std::map< FaceLocIdx, std::vector< NodeLocIdx > > f2n; + std::map< FaceLocIdx, std::vector< EdgeLocIdx > > f2e; + for( auto const & [fgi, edgeInfos]: graph.f2e ) { - for( auto const & [fgi, edgeInfos]: graph.f2e ) + FaceLocIdx const & fli = numberings.fg2l.at( fgi ); + + std::vector< NodeLocIdx > & nodes = f2n[fli]; + std::vector< EdgeLocIdx > & edges = f2e[fli]; + + nodes.reserve( std::size( edgeInfos ) ); + edges.reserve( std::size( edgeInfos ) ); + + for( EdgeInfo const & edgeInfo: edgeInfos ) { - for( EdgeInfo const & edgeInfo: edgeInfos ) - { - e2f[numberings.eg2l.at( edgeInfo.index )].emplace_back( numberings.fg2l.at( fgi ) ); - } + std::tuple< NodeGlbIdx, NodeGlbIdx > const & ngis = graph.e2n.at( edgeInfo.index ); + nodes.emplace_back( edgeInfo.start == 0 ? numberings.ng2l.at( std::get< 0 >( ngis ) ) : numberings.ng2l.at( std::get< 1 >( ngis ) ) ); + + edges.emplace_back( numberings.eg2l.at( edgeInfo.index ) ); } } +} - return makeFlavorLessEdgeMgrImpl( numEdges, ghostRank, e2n, e2f, numberings.eg2l, numberings.el2g ); + +MeshGraph mergeMeshGraph( MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts ) +{ + MeshGraph result{ owned }; + for( MeshGraph const & graph: { present, ghosts } ) + { + result.c2f.insert( std::cbegin( graph.c2f ), std::cend( graph.c2f ) ); + result.f2e.insert( std::cbegin( graph.f2e ), std::cend( graph.f2e ) ); + result.e2n.insert( std::cbegin( graph.e2n ), std::cend( graph.e2n ) ); + result.n.insert( std::cbegin( graph.n ), std::cend( graph.n ) ); + } + return result; } void buildPods( MeshGraph const & owned, @@ -199,7 +234,9 @@ void buildPods( MeshGraph const & owned, GhostRecv const & recv, GhostSend const & send ) { - GlobalNumberings const numberings = buildL2GMappings( owned, present, ghosts ); + MeshGraph const graph = mergeMeshGraph( owned, present, ghosts ); + + GlobalNumberings const numberings = buildL2GMappings( graph ); // GEOS_LOG_RANK( "numberings ng2l = " << json( numberings.ng2l ) ); // GEOS_LOG_RANK( "owned mesh graph nodes = " << json( owned.n ) ); // GEOS_LOG_RANK( "present mesh graph nodes = " << json( present.n ) ); @@ -211,7 +248,8 @@ void buildPods( MeshGraph const & owned, // MpiWrapper::barrier(); - buildEdgeMgr( numberings, owned, present, ghosts, recv, send ); + buildEdgeMgr( numberings, graph, recv, send ); + buildFaceMgr( numberings, graph, recv, send ); } } diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index 15279daa325..d9af664434f 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -59,7 +59,7 @@ struct EdgeInfo struct MeshGraph { - std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; // TODO What about the metadata (e.g. flip the face) + std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; std::map< FaceGlbIdx, std::vector< EdgeInfo > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? std::set< NodeGlbIdx > n; diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 22720e3dd96..dd0bbeca58c 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -835,7 +835,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph missingMappings.FillComplete( ownedMap, missingIndicesMap ); EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingMappings ); - MeshGraph ghosts; // TODO most likely this will be wrong. + MeshGraph ghosts; int ext = 0; std::vector< double > extValues; From 5c1e718405101e534c8d7168c95bf110f2d7469b Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:05:52 -0700 Subject: [PATCH 082/106] Change the `start` of the EdgeInfo. --- src/coreComponents/mesh/generators/NewGhosting.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index dd0bbeca58c..edc93af5ff1 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -206,7 +206,7 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > NodeGlbIdx const & n0 = face[i], & n1 = face[i + 1]; std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); - EdgeInfo const info = { n2e.at( p1 ), std::uint8_t{ p0 == p1 } }; + EdgeInfo const info = { n2e.at( p1 ), std::uint8_t{ p0 != p1 } }; f2e[fgi].emplace_back( info ); } ++fgi; From 31367df97c56f970b3318487f4d1538ae009a066 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:55:50 -0700 Subject: [PATCH 083/106] Start the work for Cell mappings. --- .../mesh/generators/BuildPods.cpp | 98 ++++++++++++++++--- .../mesh/generators/NewGlobalNumbering.cpp | 2 +- src/coreComponents/mesh/generators/Pods.cpp | 52 +++++++++- src/coreComponents/mesh/generators/Pods.hpp | 26 +++++ 4 files changed, 165 insertions(+), 13 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 19db620ddcf..1f0d6517ce8 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -24,6 +24,7 @@ namespace geos::ghosting { +// TODO Do we want 2 separated `g2l` and `l2g` structs? struct GlobalNumberings { std::map< NodeGlbIdx, NodeLocIdx > ng2l; @@ -96,6 +97,22 @@ std::vector< integer > buildGhostRank( std::map< GI, LI > const & g2l, return ghostRank; } + +MeshGraph mergeMeshGraph( MeshGraph const & owned, + MeshGraph const & present, + MeshGraph const & ghosts ) +{ + MeshGraph result{ owned }; + for( MeshGraph const & graph: { present, ghosts } ) + { + result.c2f.insert( std::cbegin( graph.c2f ), std::cend( graph.c2f ) ); + result.f2e.insert( std::cbegin( graph.f2e ), std::cend( graph.f2e ) ); + result.e2n.insert( std::cbegin( graph.e2n ), std::cend( graph.e2n ) ); + result.n.insert( std::cbegin( graph.n ), std::cend( graph.n ) ); + } + return result; +} + EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::vector< integer > const & ghostRank, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, @@ -212,22 +229,80 @@ void buildFaceMgr( GlobalNumberings const & numberings, } } - -MeshGraph mergeMeshGraph( MeshGraph const & owned, - MeshGraph const & present, - MeshGraph const & ghosts ) +std::vector< NodeLocIdx > resetFaceNodes( std::vector< NodeLocIdx > const & nodes, + bool const & isFlipped, + std::uint8_t const & start ) { - MeshGraph result{ owned }; - for( MeshGraph const & graph: { present, ghosts } ) + std::vector< NodeLocIdx > result( nodes ); + std::rotate( std::begin( result ), std::begin( result ) + start, std::end( result ) ); + if( isFlipped ) // TODO before or after? { - result.c2f.insert( std::cbegin( graph.c2f ), std::cend( graph.c2f ) ); - result.f2e.insert( std::cbegin( graph.f2e ), std::cend( graph.f2e ) ); - result.e2n.insert( std::cbegin( graph.e2n ), std::cend( graph.e2n ) ); - result.n.insert( std::cbegin( graph.n ), std::cend( graph.n ) ); + std::reverse( std::begin( result ), std::end( result ) ); } return result; } +void buildCellBlock( GlobalNumberings const & numberings, + MeshGraph const & graph, + GhostRecv const & recv, + GhostSend const & send, + std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const & f2e, + std::map< FaceLocIdx, std::vector< NodeLocIdx > > const & f2n ) +{ + // TODO MISSING cell type. Should be OK, the information is conveyed. + // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock + + // Total number of faces available in the rank (including the ghosted edges). + std::size_t const numElements = std::size( numberings.cg2l ); + + std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f; + std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; + std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n; + + for( auto const & [cgi, faceInfos]: graph.c2f ) + { + CellLocIdx const & cli = numberings.cg2l.at( cgi ); + + std::vector< FaceLocIdx > & faces = c2f[cli]; + std::vector< EdgeLocIdx > & edges = c2e[cli]; +// std::vector< NodeLocIdx > & nodes = c2n[cli]; + + faces.reserve( std::size( faceInfos ) ); + edges.reserve( std::size( faceInfos ) ); +// nodes.reserve( std::size( faceInfos ) ); + + // c2f + for( FaceInfo const & faceInfo: faceInfos ) + { + faces.emplace_back( numberings.fg2l.at( faceInfo.index ) ); + } + + // c2e + std::set< EdgeLocIdx > tmpEdges; + for( FaceLocIdx const & fli: faces ) + { + std::vector< EdgeLocIdx > const & es = f2e.at( fli ); + tmpEdges.insert( std::cbegin( es ), std::cend( es ) ); + } + edges.assign( std::cbegin( tmpEdges ), std::cend( tmpEdges ) ); + + // c2n + FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type. + FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7) + + std::vector< NodeLocIdx > const & bottomNodes = f2n.at( numberings.fg2l.at( bottomFace.index ) ); + std::vector< NodeLocIdx > const & topNodes = f2n.at( numberings.fg2l.at( topFace.index ) ); + + std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); + std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); + + // TODO carefully check the ordering... + c2n[cli] = { bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; +// nodes.insert( std::end( nodes ), std::cbegin( bn ), std::cend( bn ) ); +// nodes.insert( std::end( nodes ), std::cbegin( tn ), std::cend( tn ) ); + } +} + void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, @@ -247,7 +322,8 @@ void buildPods( MeshGraph const & owned, // auto const EdgeMgr = buildEdgeMgr( owned, present, ghosts, recv, send ); // MpiWrapper::barrier(); - + std::map< FaceLocIdx, std::vector< NodeLocIdx > > const f2n; + std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const f2e; buildEdgeMgr( numberings, graph, recv, send ); buildFaceMgr( numberings, graph, recv, send ); } diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp index aaee021b184..8b573a7ca11 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -128,7 +128,7 @@ Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes, bool & isFlipped start = intConv< std::uint8_t >( minIdx ); isFlipped = increment < 0; - integer i = intConv< integer >( minIdx ); + integer i = intConv< integer >( minIdx ); // TODO use std::rotate/std::reverse instead. for( std::size_t count = 0; count < n; ++count, i = i + increment ) { f.emplace_back( nodes.at( modulo( i ) ) ); diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 3552b87cd85..ff8230a84cf 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -119,4 +119,54 @@ ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const return {}; } -} // geos \ No newline at end of file +ElementType CellBlockImpl::getElementType() const +{ + return ElementType::Hexahedron; +} + +localIndex CellBlockImpl::numNodesPerElement() const +{ + return 0; +} + +localIndex CellBlockImpl::numEdgesPerElement() const +{ + return 0; +} + +localIndex CellBlockImpl::numFacesPerElement() const +{ + return 0; +} + +localIndex CellBlockImpl::numElements() const +{ + return 0; +} + +array2d< localIndex, cells::NODE_MAP_PERMUTATION > CellBlockImpl::getElemToNodes() const +{ + return {}; +} + +array2d< localIndex > CellBlockImpl::getElemToEdges() const +{ + return {}; +} + +array2d< localIndex > CellBlockImpl::getElemToFaces() const +{ + return {}; +} + +array1d< globalIndex > CellBlockImpl::localToGlobalMap() const +{ + return {}; +} + +std::list< dataRepository::WrapperBase const * > CellBlockImpl::getExternalProperties() const +{ + return {}; +} + +} // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 7562ae8a5cc..4855c34b311 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -21,6 +21,7 @@ #include "include/EdgeMgr.hpp" #include "include/FaceMgr.hpp" #include "include/CellMgr.hpp" +#include "include/CellBlockABC.hpp" namespace geos { @@ -95,6 +96,31 @@ class FaceMgrImpl : public generators::FaceMgr FaceLocIdx m_numFaces; }; +class CellBlockImpl: public CellBlockABC +{ +public: + ElementType getElementType() const override; + + localIndex numNodesPerElement() const override; + + localIndex numEdgesPerElement() const override; + + localIndex numFacesPerElement() const override; + + localIndex numElements() const override; + + array2d< localIndex, cells::NODE_MAP_PERMUTATION > getElemToNodes() const override; + + array2d< localIndex > getElemToEdges() const override; + + array2d< localIndex > getElemToFaces() const override; + + array1d< globalIndex > localToGlobalMap() const override; + +private: + virtual std::list< dataRepository::WrapperBase const * > getExternalProperties() const override; +}; + } // geos #endif //GEOS_PODS_HPP From 6ab3165f2ade19ed336a1e047d34d3db2e340471 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 4 Jun 2024 15:34:27 -0700 Subject: [PATCH 084/106] Add a CellBlk, unit test the face reordering, continue filling the impls. --- src/coreComponents/mesh/CMakeLists.txt | 1 + .../mesh/generators/BuildPods.cpp | 348 +++++++++++++----- .../mesh/generators/NewGlobalNumbering.cpp | 45 ++- .../mesh/generators/NewGlobalNumbering.hpp | 3 + src/coreComponents/mesh/generators/Pods.cpp | 136 +++++-- src/coreComponents/mesh/generators/Pods.hpp | 70 +++- .../mesh/generators/include/CellBlk.hpp | 88 +++++ .../mesh/generators/include/FaceMgr.hpp | 4 + .../mesh/unitTests/CMakeLists.txt | 5 +- .../mesh/unitTests/testReorderFaceNodes.cpp | 66 ++++ 10 files changed, 592 insertions(+), 174 deletions(-) create mode 100644 src/coreComponents/mesh/generators/include/CellBlk.hpp create mode 100644 src/coreComponents/mesh/unitTests/testReorderFaceNodes.cpp diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index eb19c913718..c62b41f2974 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -60,6 +60,7 @@ set( mesh_headers generators/PrismUtilities.hpp generators/WellGeneratorABC.hpp generators/WellGeneratorBase.hpp + generators/include/CellBlk.hpp generators/include/CellBlockABC.hpp generators/include/FaceBlockABC.hpp generators/include/LineBlockABC.hpp diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 1f0d6517ce8..072754e50f0 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -16,6 +16,8 @@ #include "Pods.hpp" +#include "NewGlobalNumbering.hpp" + //#include "common/MpiWrapper.hpp" #include "codingUtilities/Utilities.hpp" @@ -113,6 +115,86 @@ MeshGraph mergeMeshGraph( MeshGraph const & owned, return result; } +struct DownwardMappings +{ + std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > e2n; + std::map< FaceLocIdx, std::vector< NodeLocIdx > > f2n; + std::map< FaceLocIdx, std::vector< EdgeLocIdx > > f2e; + std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n; + std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f; + std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; +}; + + +struct UpwardMappings +{ + std::map< EdgeLocIdx, std::vector< FaceLocIdx > > e2f; + std::map< FaceLocIdx, std::vector< CellLocIdx > > f2c; + std::map< NodeLocIdx, std::vector< EdgeLocIdx > > n2e; + std::map< NodeLocIdx, std::vector< FaceLocIdx > > n2f; + std::map< NodeLocIdx, std::vector< CellLocIdx > > n2c; +}; + +template< class T, class U > +ArrayOfArrays< localIndex > convertToAoA( std::map< T, std::vector< U > > const & t2u ) +{ + static_assert( !std::is_same_v< T, U > ); + ArrayOfArrays< localIndex > t2u_; + + std::size_t const numTs = std::size( t2u ); + + std::vector< int > sizes; + sizes.reserve( numTs ); + for( auto const & [_, us]: t2u ) + { + sizes.emplace_back( std::size( us ) ); + } + t2u_.resizeFromCapacities< serialPolicy >( numTs, sizes.data() ); + for( auto const & [t, us]: t2u ) + { + for( U const & u: us ) + { + t2u_.emplaceBack( t.get(), u.get() ); + } + } + + return t2u_; +} + +template< class T, class U > +array2d< localIndex > convertToA2d( std::map< T, std::vector< U > > const & t2u, + int dimU ) +{ + static_assert( !std::is_same_v< T, U > ); + array2d< localIndex > t2u_( std::size( t2u ), dimU ); + t2u_.setValues< serialPolicy >( -1 ); + + for( auto const & [t, us]: t2u ) + { + for( int i = 0; i < intConv< int >( std::size( us ) ); ++i ) + { + t2u_( t.get(), i ) = us[i].get(); + } + } + return t2u_; +} + +template< class GI, class LI > +std::tuple< array1d< globalIndex >, unordered_map< globalIndex, localIndex > > +convertGlbLoc( std::map< GI, LI > const & g2l ) +{ + static_assert( !std::is_same_v< GI, LI > ); + array1d< globalIndex > l2g_( std::size( g2l ) ); + unordered_map< globalIndex, localIndex > g2l_; + for( auto const & [gi, li]: g2l ) + { + l2g_[li.get()] = gi.get(); + g2l_[gi.get()] = li.get(); + } + + return { l2g_, g2l_ }; +} + EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::vector< integer > const & ghostRank, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, @@ -128,93 +210,100 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, e2n_[i][1] = std::get< 1 >( nlis ).get(); } - ArrayOfArrays< localIndex > e2f_; - std::vector< int > sizes; - sizes.reserve( numEdges ); - for( auto const & [_, flis]: e2f ) - { - sizes.emplace_back( std::size( flis ) ); - } - GEOS_ASSERT_EQ( std::size( sizes ), numEdges ); - e2f_.resizeFromCapacities< serialPolicy >( numEdges, sizes.data() ); - for( auto const & [eli, flis]: e2f ) - { - for( FaceLocIdx const & fli: flis ) - { - e2f_.emplaceBack( eli.get(), fli.get() ); - } - } + auto [l2g, g2l] = convertGlbLoc( eg2l ); - array1d< globalIndex > l2g( numEdges ); - unordered_map< globalIndex, localIndex > g2l; - for( auto const & [egi, eli]: eg2l ) + array1d< integer > ghostRank_( numEdges ); + for( integer const & gr: ghostRank ) { - l2g[eli.get()] = egi.get(); - g2l[egi.get()] = eli.get(); + ghostRank_.emplace_back( gr ); } - array1d< integer > ghostRank_( numEdges ); + return EdgeMgrImpl( numEdges, std::move( ghostRank_ ), std::move( e2n_ ), convertToAoA( e2f ), std::move( g2l ), std::move( l2g ) ); +} + +FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, + std::vector< integer > const & ghostRank, + std::map< FaceLocIdx, std::vector< NodeLocIdx > > const & f2n, + std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const & f2e, + std::map< FaceLocIdx, std::vector< CellLocIdx > > const & f2c, + std::map< FaceGlbIdx, FaceLocIdx > const & fg2l, + std::map< FaceLocIdx, FaceGlbIdx > const & fl2g ) +{ + auto [l2g, g2l] = convertGlbLoc( fg2l ); + + array1d< integer > ghostRank_( numFaces ); for( integer const & gr: ghostRank ) { ghostRank_.emplace_back( gr ); } - return EdgeMgrImpl( numEdges, std::move( ghostRank_ ), std::move( e2n_ ), std::move( e2f_ ), std::move( g2l ), std::move( l2g ) ); + return FaceMgrImpl( numFaces, std::move( ghostRank_ ), convertToAoA( f2n ), convertToAoA( f2e ), convertToA2d( f2c, 2 ), std::move( g2l ), std::move( l2g ) ); } -EdgeMgrImpl buildEdgeMgr( GlobalNumberings const & numberings, - MeshGraph const & graph, - GhostRecv const & recv, - GhostSend const & send ) +NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, + std::vector< integer > const & ghostRank, + std::map< NodeLocIdx, std::vector< EdgeLocIdx > > const & n2e, + std::map< NodeLocIdx, std::vector< FaceLocIdx > > const & n2f, + std::map< NodeLocIdx, std::vector< CellLocIdx > > const & n2c, + std::map< NodeGlbIdx, NodeLocIdx > const & ng2l, + std::map< NodeLocIdx, NodeGlbIdx > const & nl2g ) { - // Total number of edges available in the rank (including the ghosted edges). - std::size_t const numEdges = std::size( numberings.el2g ); - - // Building the ghost rank. - std::vector< integer > const ghostRank = buildGhostRank( numberings.eg2l, send.edges, recv.edges ); + // TODO MISSING cell type. Should be OK, the information is conveyed. + // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock + auto [l2g, g2l] = convertGlbLoc( ng2l ); - // Building the edges to nodes mapping - std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > e2n; - for( auto const & [egi, ngis]: graph.e2n ) + array1d< integer > ghostRank_( numNodes ); + for( integer const & gr: ghostRank ) { - NodeLocIdx const nli0 = numberings.ng2l.at( std::get< 0 >( ngis ) ); - NodeLocIdx const nli1 = numberings.ng2l.at( std::get< 1 >( ngis ) ); - e2n[numberings.eg2l.at( egi )] = { nli0, nli1 }; + ghostRank_.emplace_back( gr ); } - // Building the edges to nodes mapping - std::map< EdgeLocIdx, std::vector< FaceLocIdx > > e2f; - for( auto const & [fgi, edgeInfos]: graph.f2e ) + return NodeMgrImpl( intConv< localIndex >( numNodes ), std::move( ghostRank_ ), convertToAoA( n2e ), convertToAoA( n2f ), convertToAoA( n2c ), std::move( l2g ) ); +} + +CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, + std::vector< integer > const & ghostRank, + std::map< CellLocIdx, std::vector< NodeLocIdx > > const & c2n, + std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, + std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, + std::map< CellGlbIdx, CellLocIdx > const & cg2l, + std::map< CellLocIdx, CellGlbIdx > const & cl2g ) +{ + // TODO MISSING cell type. Should be OK, the information is conveyed. + // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock + auto [l2g, g2l] = convertGlbLoc( cg2l ); + + array1d< integer > ghostRank_( numCells ); + for( integer const & gr: ghostRank ) { - for( EdgeInfo const & edgeInfo: edgeInfos ) - { - e2f[numberings.eg2l.at( edgeInfo.index )].emplace_back( numberings.fg2l.at( fgi ) ); - } + ghostRank_.emplace_back( gr ); } - return makeFlavorlessEdgeMgrImpl( numEdges, ghostRank, e2n, e2f, numberings.eg2l, numberings.el2g ); + return CellBlkImpl( intConv< localIndex >( numCells ), ghostRank_, convertToA2d( c2n, 8 ), convertToA2d( c2e, 12 ), convertToA2d( c2f, 6 ), std::move( l2g ) ); } -void buildFaceMgr( GlobalNumberings const & numberings, - MeshGraph const & graph, - GhostRecv const & recv, - GhostSend const & send ) +DownwardMappings buildDownwardMappings( GlobalNumberings const & numberings, + MeshGraph const & graph, + GhostRecv const & recv, + GhostSend const & send ) { - // Total number of faces available in the rank (including the ghosted edges). - std::size_t const numFaces = std::size( numberings.fl2g ); + DownwardMappings res; - // Building the ghost rank. - std::vector< integer > const ghostRank = buildGhostRank( numberings.fg2l, send.faces, recv.faces ); + // Building the `e2n` (edges to nodes) mapping + for( auto const & [egi, ngis]: graph.e2n ) + { + NodeLocIdx const nli0 = numberings.ng2l.at( std::get< 0 >( ngis ) ); + NodeLocIdx const nli1 = numberings.ng2l.at( std::get< 1 >( ngis ) ); + res.e2n[numberings.eg2l.at( egi )] = { nli0, nli1 }; + } - // Building the `f2n` and `f2e` mappings - std::map< FaceLocIdx, std::vector< NodeLocIdx > > f2n; - std::map< FaceLocIdx, std::vector< EdgeLocIdx > > f2e; + // Building the `f2n` (face to nodes) and `f2e` (faces to edges) mappings for( auto const & [fgi, edgeInfos]: graph.f2e ) { FaceLocIdx const & fli = numberings.fg2l.at( fgi ); - std::vector< NodeLocIdx > & nodes = f2n[fli]; - std::vector< EdgeLocIdx > & edges = f2e[fli]; + std::vector< NodeLocIdx > & nodes = res.f2n[fli]; + std::vector< EdgeLocIdx > & edges = res.f2e[fli]; nodes.reserve( std::size( edgeInfos ) ); edges.reserve( std::size( edgeInfos ) ); @@ -227,49 +316,17 @@ void buildFaceMgr( GlobalNumberings const & numberings, edges.emplace_back( numberings.eg2l.at( edgeInfo.index ) ); } } -} - -std::vector< NodeLocIdx > resetFaceNodes( std::vector< NodeLocIdx > const & nodes, - bool const & isFlipped, - std::uint8_t const & start ) -{ - std::vector< NodeLocIdx > result( nodes ); - std::rotate( std::begin( result ), std::begin( result ) + start, std::end( result ) ); - if( isFlipped ) // TODO before or after? - { - std::reverse( std::begin( result ), std::end( result ) ); - } - return result; -} - -void buildCellBlock( GlobalNumberings const & numberings, - MeshGraph const & graph, - GhostRecv const & recv, - GhostSend const & send, - std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const & f2e, - std::map< FaceLocIdx, std::vector< NodeLocIdx > > const & f2n ) -{ - // TODO MISSING cell type. Should be OK, the information is conveyed. - // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock - - // Total number of faces available in the rank (including the ghosted edges). - std::size_t const numElements = std::size( numberings.cg2l ); - - std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f; - std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; - std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n; + // Building the `c2n` (cell to nodes), `c2e` (cell to edges) and `c2f` (cell to faces) mappings for( auto const & [cgi, faceInfos]: graph.c2f ) { CellLocIdx const & cli = numberings.cg2l.at( cgi ); - std::vector< FaceLocIdx > & faces = c2f[cli]; - std::vector< EdgeLocIdx > & edges = c2e[cli]; -// std::vector< NodeLocIdx > & nodes = c2n[cli]; + std::vector< FaceLocIdx > & faces = res.c2f[cli]; + std::vector< EdgeLocIdx > & edges = res.c2e[cli]; faces.reserve( std::size( faceInfos ) ); edges.reserve( std::size( faceInfos ) ); -// nodes.reserve( std::size( faceInfos ) ); // c2f for( FaceInfo const & faceInfo: faceInfos ) @@ -281,7 +338,7 @@ void buildCellBlock( GlobalNumberings const & numberings, std::set< EdgeLocIdx > tmpEdges; for( FaceLocIdx const & fli: faces ) { - std::vector< EdgeLocIdx > const & es = f2e.at( fli ); + std::vector< EdgeLocIdx > const & es = res.f2e.at( fli ); tmpEdges.insert( std::cbegin( es ), std::cend( es ) ); } edges.assign( std::cbegin( tmpEdges ), std::cend( tmpEdges ) ); @@ -290,17 +347,74 @@ void buildCellBlock( GlobalNumberings const & numberings, FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type. FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7) - std::vector< NodeLocIdx > const & bottomNodes = f2n.at( numberings.fg2l.at( bottomFace.index ) ); - std::vector< NodeLocIdx > const & topNodes = f2n.at( numberings.fg2l.at( topFace.index ) ); + std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( numberings.fg2l.at( bottomFace.index ) ); + std::vector< NodeLocIdx > const & topNodes = res.f2n.at( numberings.fg2l.at( topFace.index ) ); std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); // TODO carefully check the ordering... - c2n[cli] = { bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; + res.c2n[cli] = { bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; // nodes.insert( std::end( nodes ), std::cbegin( bn ), std::cend( bn ) ); // nodes.insert( std::end( nodes ), std::cbegin( tn ), std::cend( tn ) ); } + + return res; +} + +/** + * @brief Simple inversions of the downward mappings. + * @param downwardMappings The downward mappings. + * @return The mappings. + */ +UpwardMappings buildUpwardMappings( DownwardMappings const & downwardMappings ) +{ + UpwardMappings res; + + // Filling `e2f` + for( auto const & [fli, elis]: downwardMappings.f2e ) + { + for( EdgeLocIdx const & eli: elis ) + { + res.e2f[eli].emplace_back( fli ); + } + } + + // Filling `f2c` + for( auto const & [cli, flis]: downwardMappings.c2f ) + { + for( FaceLocIdx const & fli: flis ) + { + res.f2c[fli].emplace_back( cli ); + } + } + + // Filling `n2e` + for( auto const & [eli, nlis]: downwardMappings.e2n ) + { + res.n2e[std::get< 0 >( nlis )].emplace_back( eli ); + res.n2e[std::get< 1 >( nlis )].emplace_back( eli ); + } + + // Filling `n2f` + for( auto const & [fli, nlis]: downwardMappings.f2n ) + { + for( NodeLocIdx const & nli: nlis ) + { + res.n2f[nli].emplace_back( fli ); + } + } + + // Filling `n2c` + for( auto const & [cli, nlis]: downwardMappings.c2n ) + { + for( NodeLocIdx const & nli: nlis ) + { + res.n2c[nli].emplace_back( cli ); + } + } + + return res; } void buildPods( MeshGraph const & owned, @@ -322,10 +436,40 @@ void buildPods( MeshGraph const & owned, // auto const EdgeMgr = buildEdgeMgr( owned, present, ghosts, recv, send ); // MpiWrapper::barrier(); - std::map< FaceLocIdx, std::vector< NodeLocIdx > > const f2n; - std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const f2e; - buildEdgeMgr( numberings, graph, recv, send ); - buildFaceMgr( numberings, graph, recv, send ); + + DownwardMappings const downwardMappings = buildDownwardMappings( numberings, graph, recv, send ); + UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); + + NodeMgrImpl const nodeMgr = makeFlavorlessNodeMgrImpl( std::size( numberings.ng2l ), + buildGhostRank( numberings.ng2l, send.nodes, recv.nodes ), + upwardMappings.n2e, + upwardMappings.n2f, + upwardMappings.n2c, + numberings.ng2l, + numberings.nl2g ); + + EdgeMgrImpl const edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( numberings.eg2l ), + buildGhostRank( numberings.eg2l, send.edges, recv.edges ), + downwardMappings.e2n, + upwardMappings.e2f, + numberings.eg2l, + numberings.el2g ); + + FaceMgrImpl const faceMgr = makeFlavorlessFaceMgrImpl( std::size( numberings.fg2l ), + buildGhostRank( numberings.fg2l, send.faces, recv.faces ), + downwardMappings.f2n, + downwardMappings.f2e, + upwardMappings.f2c, + numberings.fg2l, + numberings.fl2g ); + + CellBlkImpl const cellBlock = makeFlavorlessCellBlkImpl( std::size( numberings.cg2l ), + buildGhostRank( numberings.cg2l, send.cells, recv.cells ), + downwardMappings.c2n, + downwardMappings.c2e, + downwardMappings.c2f, + numberings.cg2l, + numberings.cl2g ); } } diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp index 8b573a7ca11..62d6c34e61d 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -104,39 +104,44 @@ Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes, bool & isFlipped { std::size_t const n = nodes.size(); - // Handles negative values of `i`. - auto const modulo = [n]( integer const & i ) -> std::size_t - { - integer mod = i % n; - if( mod < 0 ) - { - mod += n; - } - return mod; - }; - - Face f; - f.reserve( n ); + std::vector< NodeGlbIdx > f( nodes ); auto const cit = std::min_element( std::cbegin( nodes ), std::cend( nodes ) ); auto const minIdx = std::distance( std::cbegin( nodes ), cit ); - std::size_t const prevIdx = modulo( intConv< integer >( minIdx - 1 ) ); - std::size_t const nextIdx = modulo( intConv< integer >( minIdx + 1 ) ); - int const increment = nodes[prevIdx] < nodes[nextIdx] ? -1 : 1; + std::size_t const prevIdx = minIdx == 0 ? n - 1 : minIdx - 1; + std::size_t const nextIdx = minIdx == intConv< int >( n ) - 1 ? 0 : minIdx + 1; start = intConv< std::uint8_t >( minIdx ); - isFlipped = increment < 0; + isFlipped = nodes[prevIdx] < nodes[nextIdx]; - integer i = intConv< integer >( minIdx ); // TODO use std::rotate/std::reverse instead. - for( std::size_t count = 0; count < n; ++count, i = i + increment ) + std::size_t const pivot = isFlipped ? n - 1 - minIdx : minIdx; + if( isFlipped ) { - f.emplace_back( nodes.at( modulo( i ) ) ); + std::reverse( std::begin( f ), std::end( f ) ); } + std::rotate( std::begin( f ), std::begin( f ) + pivot, std::end( f ) ); return f; } +std::vector< NodeLocIdx > resetFaceNodes( std::vector< NodeLocIdx > const & nodes, + bool const & isFlipped, + std::uint8_t const & start ) +{ + std::vector< NodeLocIdx > result( nodes ); + if( isFlipped ) + { + std::reverse( std::begin( result ), std::end( result ) ); + std::uint8_t const pivot = std::size( result ) - 1 - start; + std::rotate( std::begin( result ), std::begin( result ) + pivot, std::end( result ) ); + } + else + { + std::rotate( std::rbegin( result ), std::rbegin( result ) + start, std::rend( result ) ); + } + return result; +} Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, std::set< vtkIdType > const & cellIds ) diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp index 118f5d4f45d..6d73f6f4227 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.hpp @@ -84,6 +84,9 @@ inline void from_json( const json & j, */ Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes, bool & isFlipped, std::uint8_t & start ); +std::vector< NodeLocIdx > resetFaceNodes( std::vector< NodeLocIdx > const & nodes, + bool const & isFlipped, + std::uint8_t const & start ); std::tuple< Buckets, BucketOffsets > doTheNewGlobalNumbering( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors ); diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index ff8230a84cf..961eac8d05a 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -17,38 +17,59 @@ namespace geos { -NodeMgrImpl::NodeMgrImpl( NodeLocIdx const & numNodes ) - : m_numNodes( numNodes ) +NodeMgrImpl::NodeMgrImpl( localIndex numNodes, + array1d< integer > && ghostRank, + ArrayOfArrays< localIndex > const & n2e, + ArrayOfArrays< localIndex > const & n2f, + ArrayOfArrays< localIndex > const & n2c, + array1d< globalIndex > const & l2g ) + : m_numNodes( numNodes ), + m_ghostRank( ghostRank ), + m_n2e( n2e ), + m_n2f( n2f ), + m_n2c( n2c ), + m_l2g( l2g ) { } localIndex NodeMgrImpl::numNodes() const { - return intConv< localIndex >( m_numNodes.get() ); + return m_numNodes; } array2d< real64, nodes::REFERENCE_POSITION_PERM > NodeMgrImpl::getNodePositions() const { - return {}; + return {}; } ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToEdges() const { - return {}; + return m_n2e; } ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToFaces() const { - return {}; + return m_n2f; } ToCellRelation< ArrayOfArrays< localIndex > > NodeMgrImpl::getNodeToElements() const { - return {}; + ArrayOfArrays< localIndex > toBlockIndex( m_n2c ); // TODO cheat in case there's one unique cell block! + for( int i = 0; i < toBlockIndex.size(); ++i ) + { + for( int j = 0; j < toBlockIndex.sizeOfArray(i); ++j ) + { + if( m_n2c( i, j ) > -1 ) + { + toBlockIndex( i, j ) = 0; + } + } + } + return { std::move( toBlockIndex ), m_n2c }; } array1d< globalIndex > NodeMgrImpl::getLocalToGlobal() const { - return {}; + return m_l2g; } std::map< string, SortedArray< localIndex > > const & NodeMgrImpl::getNodeSets() const @@ -95,78 +116,125 @@ array1d< globalIndex > EdgeMgrImpl::getLocalToGlobal() const return m_l2g; } -FaceMgrImpl::FaceMgrImpl( FaceLocIdx const & numFaces ) - : m_numFaces( numFaces ) +FaceMgrImpl::FaceMgrImpl( std::size_t numFaces, + array1d< integer > && ghostRank, + ArrayOfArrays< localIndex > && f2n, + ArrayOfArrays< localIndex > && f2e, + array2d< localIndex > && f2c, + unordered_map< globalIndex, localIndex > && g2l, + array1d< globalIndex > && l2g ) + : m_numFaces( numFaces ), + m_ghostRank( ghostRank ), + m_f2n( f2n ), + m_f2e( f2e ), + m_f2c( f2c ), + m_g2l( g2l ), + m_l2g( l2g ) { } localIndex FaceMgrImpl::numFaces() const { - return intConv< localIndex >( m_numFaces.get() ); + return intConv< localIndex >( m_numFaces ); } ArrayOfArrays< localIndex > FaceMgrImpl::getFaceToNodes() const { - return {}; + return m_f2n; } ArrayOfArrays< localIndex > FaceMgrImpl::getFaceToEdges() const { - return {}; + return m_f2e; } ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const { - return {}; + array2d< localIndex > toBlockIndex( m_f2c ); // TODO cheat in case there's one unique cell block! + for( int i = 0; i < toBlockIndex.size( 0 ); ++i ) + { + for( int j = 0; j < toBlockIndex.size( 1 ); ++j ) + { + if( m_f2c( i, j ) > -1 ) + { + toBlockIndex( i, j ) = 0; + } + } + } + return { std::move( toBlockIndex ), m_f2c }; } -ElementType CellBlockImpl::getElementType() const +array1d< integer > FaceMgrImpl::getGhostRank() const { - return ElementType::Hexahedron; + return m_ghostRank; } -localIndex CellBlockImpl::numNodesPerElement() const +array1d< globalIndex > FaceMgrImpl::getLocalToGlobal() const { - return 0; + return m_l2g; } -localIndex CellBlockImpl::numEdgesPerElement() const +CellBlkImpl::CellBlkImpl( localIndex numCells, + array1d< integer > const & ghostRank, + array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, + array2d< localIndex > const & c2e, + array2d< localIndex > const & c2f, + array1d< globalIndex > const & l2g ) + : m_numCells( numCells ), + m_ghostRank( ghostRank ), + m_c2n( c2n ), + m_c2e( c2e ), + m_c2f( c2f ), + m_l2g( l2g ) +{ } + +ElementType CellBlkImpl::getElementType() const { - return 0; + return ElementType::Hexahedron; } -localIndex CellBlockImpl::numFacesPerElement() const +localIndex CellBlkImpl::numNodesPerElement() const { - return 0; + return 8; } -localIndex CellBlockImpl::numElements() const +localIndex CellBlkImpl::numEdgesPerElement() const { - return 0; + return 12; } -array2d< localIndex, cells::NODE_MAP_PERMUTATION > CellBlockImpl::getElemToNodes() const +localIndex CellBlkImpl::numFacesPerElement() const { - return {}; + return 6; } -array2d< localIndex > CellBlockImpl::getElemToEdges() const +localIndex CellBlkImpl::numElements() const { - return {}; + return m_numCells; } -array2d< localIndex > CellBlockImpl::getElemToFaces() const +array2d< localIndex, cells::NODE_MAP_PERMUTATION > CellBlkImpl::getElemToNodes() const { - return {}; + return m_c2n; } -array1d< globalIndex > CellBlockImpl::localToGlobalMap() const +array2d< localIndex > CellBlkImpl::getElemToEdges() const { - return {}; + return m_c2e; } -std::list< dataRepository::WrapperBase const * > CellBlockImpl::getExternalProperties() const +array2d< localIndex > CellBlkImpl::getElemToFaces() const { - return {}; + return m_c2f; +} + +array1d< globalIndex > CellBlkImpl::localToGlobalMap() const +{ + return m_l2g; } +//std::list< dataRepository::WrapperBase const * > CellBlkImpl::getExternalProperties() const +//{ +// return {}; +//} + } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 4855c34b311..47ddf4c571d 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -21,7 +21,7 @@ #include "include/EdgeMgr.hpp" #include "include/FaceMgr.hpp" #include "include/CellMgr.hpp" -#include "include/CellBlockABC.hpp" +#include "include/CellBlk.hpp" namespace geos { @@ -29,7 +29,12 @@ namespace geos class NodeMgrImpl : public generators::NodeMgr { public: - explicit NodeMgrImpl( NodeLocIdx const & numNodes ); + NodeMgrImpl( localIndex numNodes, + array1d< integer > && ghostRank, + ArrayOfArrays< localIndex > const & n2e, + ArrayOfArrays< localIndex > const & n2f, + ArrayOfArrays< localIndex > const & n2c, + array1d< globalIndex > const & l2g ); [[nodiscard]] localIndex numNodes() const override; @@ -46,7 +51,12 @@ class NodeMgrImpl : public generators::NodeMgr [[nodiscard]] std::map< string, SortedArray< localIndex > > const & getNodeSets() const override; private: - NodeLocIdx m_numNodes; + localIndex m_numNodes; + array1d< integer > m_ghostRank; + ArrayOfArrays< localIndex > m_n2e; + ArrayOfArrays< localIndex > m_n2f; + ArrayOfArrays< localIndex > m_n2c; + array1d< globalIndex > m_l2g; std::map< string, SortedArray< localIndex > > m_todo; }; @@ -82,7 +92,13 @@ class EdgeMgrImpl : public generators::EdgeMgr class FaceMgrImpl : public generators::FaceMgr { public: - explicit FaceMgrImpl( FaceLocIdx const & numFaces ); + FaceMgrImpl( std::size_t numFaces, + array1d< integer > && ghostRank, + ArrayOfArrays< localIndex > && f2n, + ArrayOfArrays< localIndex > && f2e, + array2d< localIndex > && f2c, + unordered_map< globalIndex, localIndex > && g2l, + array1d< globalIndex > && l2g ); [[nodiscard]] localIndex numFaces() const override; @@ -92,33 +108,55 @@ class FaceMgrImpl : public generators::FaceMgr [[nodiscard]] ToCellRelation< array2d< localIndex > > getFaceToElements() const override; + [[nodiscard]] array1d< integer > getGhostRank() const override; + + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; + private: - FaceLocIdx m_numFaces; + localIndex m_numFaces; + array1d< integer > m_ghostRank; + ArrayOfArrays< localIndex > m_f2n; + ArrayOfArrays< localIndex > m_f2e; + array2d< localIndex > m_f2c; + unordered_map< globalIndex, localIndex > m_g2l; + array1d< globalIndex > m_l2g; }; -class CellBlockImpl: public CellBlockABC +class CellBlkImpl : public CellBlk { public: - ElementType getElementType() const override; + CellBlkImpl( localIndex numCells, + array1d< integer > const & ghostRank, + array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, + array2d< localIndex > const & c2e, + array2d< localIndex > const & c2f, + array1d< globalIndex > const & l2g ); + + [[nodiscard]] ElementType getElementType() const override; - localIndex numNodesPerElement() const override; + [[nodiscard]] localIndex numNodesPerElement() const override; - localIndex numEdgesPerElement() const override; + [[nodiscard]] localIndex numEdgesPerElement() const override; - localIndex numFacesPerElement() const override; + [[nodiscard]] localIndex numFacesPerElement() const override; - localIndex numElements() const override; + [[nodiscard]] localIndex numElements() const override; - array2d< localIndex, cells::NODE_MAP_PERMUTATION > getElemToNodes() const override; + [[nodiscard]] array2d< localIndex, cells::NODE_MAP_PERMUTATION > getElemToNodes() const override; - array2d< localIndex > getElemToEdges() const override; + [[nodiscard]] array2d< localIndex > getElemToEdges() const override; - array2d< localIndex > getElemToFaces() const override; + [[nodiscard]] array2d< localIndex > getElemToFaces() const override; - array1d< globalIndex > localToGlobalMap() const override; + [[nodiscard]] array1d< globalIndex > localToGlobalMap() const override; private: - virtual std::list< dataRepository::WrapperBase const * > getExternalProperties() const override; + localIndex m_numCells; + array1d< integer > m_ghostRank; + array2d< localIndex, cells::NODE_MAP_PERMUTATION > m_c2n; + array2d< localIndex > m_c2e; + array2d< localIndex > m_c2f; + array1d< globalIndex > m_l2g; }; } // geos diff --git a/src/coreComponents/mesh/generators/include/CellBlk.hpp b/src/coreComponents/mesh/generators/include/CellBlk.hpp new file mode 100644 index 00000000000..57fd2d1d52b --- /dev/null +++ b/src/coreComponents/mesh/generators/include/CellBlk.hpp @@ -0,0 +1,88 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 Total, S.A + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_CELLBLK_HPP +#define GEOS_CELLBLK_HPP + +#include "mesh/ElementType.hpp" +#include "common/DataTypes.hpp" + + +namespace geos +{ + +class CellBlk +{ +public: + + /** + * @brief Get the type of element in this subregion. + * @return a string specifying the type of element in this subregion + * + * See class FiniteElementBase for possible element type. + */ + [[nodiscard]] virtual ElementType getElementType() const = 0; + + /** + * @brief Get the number of nodes per element. + * @return number of nodes per element + */ + [[nodiscard]] virtual localIndex numNodesPerElement() const = 0; + + /** + * @brief Get the number of edges per element. + * @return number of edges per element + */ + [[nodiscard]] virtual localIndex numEdgesPerElement() const = 0; + + /** + * @brief Get the number of faces per element. + * @return number of faces per element + */ + [[nodiscard]] virtual localIndex numFacesPerElement() const = 0; + + /** + * @brief Get the number of elements. + * @return number of elements in the cell block + */ + [[nodiscard]] virtual localIndex numElements() const = 0; + + /** + * @brief Get the element-to-nodes map. + * @return The mapping relationship as a 2d-array. + */ + [[nodiscard]] virtual array2d< localIndex, cells::NODE_MAP_PERMUTATION > getElemToNodes() const = 0; + + /** + * @brief Get the element-to-edges map. + * @return The mapping relationship as a 2d-array. + */ + [[nodiscard]] virtual array2d< localIndex > getElemToEdges() const = 0; + + /** + * @brief Get the element-to-faces map. + * @return The mapping relationship as a 2d-array. + */ + [[nodiscard]] virtual array2d< localIndex > getElemToFaces() const = 0; + + /** + * @brief Get local to global map. + * @return The mapping relationship as an array. + */ + [[nodiscard]] virtual array1d< globalIndex > localToGlobalMap() const = 0; +}; + +} // end of namespace + +#endif // include guard diff --git a/src/coreComponents/mesh/generators/include/FaceMgr.hpp b/src/coreComponents/mesh/generators/include/FaceMgr.hpp index 439c1e7283f..7fd0add32c8 100644 --- a/src/coreComponents/mesh/generators/include/FaceMgr.hpp +++ b/src/coreComponents/mesh/generators/include/FaceMgr.hpp @@ -50,6 +50,10 @@ class FaceMgr * In case the face only belongs to one single element, the second value of the table is -1. */ [[nodiscard]] virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const = 0; + + [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; + + [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; }; } diff --git a/src/coreComponents/mesh/unitTests/CMakeLists.txt b/src/coreComponents/mesh/unitTests/CMakeLists.txt index 862bbca219e..30a746798f7 100644 --- a/src/coreComponents/mesh/unitTests/CMakeLists.txt +++ b/src/coreComponents/mesh/unitTests/CMakeLists.txt @@ -1,8 +1,9 @@ # Specify list of tests set( mesh_tests - testMeshObjectPath.cpp testComputationalGeometry.cpp - testGeometricObjects.cpp ) + testGeometricObjects.cpp + testMeshObjectPath.cpp + testReorderFaceNodes.cpp ) set( dependencyList blas lapack gtest mesh ${parallelDeps} ) diff --git a/src/coreComponents/mesh/unitTests/testReorderFaceNodes.cpp b/src/coreComponents/mesh/unitTests/testReorderFaceNodes.cpp new file mode 100644 index 00000000000..0cf03e40e02 --- /dev/null +++ b/src/coreComponents/mesh/unitTests/testReorderFaceNodes.cpp @@ -0,0 +1,66 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "../generators/NewGlobalNumbering.hpp" + +#include + +namespace geos +{ + +struct Data +{ + std::vector< int > const input; + bool const isFlipped; + std::uint8_t const start; + std::vector< int > const expected; +}; + +class TestReorderFaceNodes : public ::testing::TestWithParam< Data > +{ + +}; + +template< class I > +std::vector< I > conv( std::vector< int > const & input ) +{ + std::vector< I > res; + res.reserve( std::size( input ) ); + for( int const & i: input ) + { + res.emplace_back( I{ i } ); + } + return res; +} + +TEST_P( TestReorderFaceNodes, BB ) +{ + Data const & d = GetParam(); + bool isFlipped; + std::uint8_t start; + std::vector< NodeGlbIdx > const res = ghosting::reorderFaceNodes( conv< NodeGlbIdx >( d.input ), isFlipped, start ); + EXPECT_EQ( std::make_tuple( conv< NodeGlbIdx >( d.expected ), d.isFlipped, d.start ), std::tie( res, isFlipped, start ) ); + EXPECT_EQ( conv< NodeLocIdx >( d.input ), ghosting::resetFaceNodes( conv< NodeLocIdx >( d.expected ), d.isFlipped, d.start ) ); +} + +INSTANTIATE_TEST_SUITE_P( instance, TestReorderFaceNodes, ::testing::Values( + Data{ { 0, 1, 2 }, false, 0, { 0, 1, 2 } }, + Data{ { 2, 0, 1 }, false, 1, { 0, 1, 2 } }, + Data{ { 2, 1, 0 }, true, 2, { 0, 1, 2 } }, + Data{ { 0, 2, 1 }, true, 0, { 0, 1, 2 } }, + Data{ { 1, 0, 2, 3 }, true, 1, { 0, 1, 3, 2 } }, + Data{ { 2, 0, 1, 3 }, false, 1, { 0, 1, 3, 2 } } +) ); + +} // end of namespace From 0ed550fe3f097660e9cd8442e2b0344e46e3b600 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 4 Jun 2024 15:46:06 -0700 Subject: [PATCH 085/106] Separating g2l and l2g. --- .../mesh/generators/BuildPods.cpp | 110 +++++++++--------- 1 file changed, 54 insertions(+), 56 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 072754e50f0..2098ae2d288 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -26,22 +26,21 @@ namespace geos::ghosting { -// TODO Do we want 2 separated `g2l` and `l2g` structs? -struct GlobalNumberings +struct GlobalToLocal { - std::map< NodeGlbIdx, NodeLocIdx > ng2l; - std::map< NodeLocIdx, NodeGlbIdx > nl2g; - - std::map< EdgeGlbIdx, EdgeLocIdx > eg2l; - std::map< EdgeLocIdx, EdgeGlbIdx > el2g; // TODO Do we want to make this a vector already? It's surely the most efficient way to build it. - - std::map< FaceGlbIdx, FaceLocIdx > fg2l; - std::map< FaceLocIdx, FaceGlbIdx > fl2g; - - std::map< CellGlbIdx, CellLocIdx > cg2l; - std::map< CellLocIdx, CellGlbIdx > cl2g; + std::map< NodeGlbIdx, NodeLocIdx > nodes; + std::map< EdgeGlbIdx, EdgeLocIdx > edges; + std::map< FaceGlbIdx, FaceLocIdx > faces; + std::map< CellGlbIdx, CellLocIdx > cells; }; +struct LocalToGlobal +{ + std::map< NodeLocIdx, NodeGlbIdx > nodes; + std::map< EdgeLocIdx, EdgeGlbIdx > edges; // TODO Do we want to make this a vector already? It's surely the most efficient way to build it. + std::map< FaceLocIdx, FaceGlbIdx > faces; + std::map< CellLocIdx, CellGlbIdx > cells; +}; template< class GI, class LI > void buildL2GMappings( std::set< GI > const & gis, @@ -62,9 +61,10 @@ void buildL2GMappings( std::set< GI > const & gis, // TODO add a check to see if we have the full range of local indices. } -GlobalNumberings buildL2GMappings( MeshGraph const & graph ) +std::tuple< GlobalToLocal, LocalToGlobal > buildL2GMappings( MeshGraph const & graph ) { - GlobalNumberings result; + GlobalToLocal g2l; + LocalToGlobal l2g; // Use `std::ranges::views::keys` when switching to C++20 auto const keys = []( auto const & m ) @@ -72,12 +72,12 @@ GlobalNumberings buildL2GMappings( MeshGraph const & graph ) return mapKeys< std::set >( m ); }; - buildL2GMappings( graph.n, result.ng2l, result.nl2g ); - buildL2GMappings( keys( graph.e2n ), result.eg2l, result.el2g ); - buildL2GMappings( keys( graph.f2e ), result.fg2l, result.fl2g ); - buildL2GMappings( keys( graph.c2f ), result.cg2l, result.cl2g ); + buildL2GMappings( graph.n, g2l.nodes, l2g.nodes ); + buildL2GMappings( keys( graph.e2n ), g2l.edges, l2g.edges ); + buildL2GMappings( keys( graph.f2e ), g2l.faces, l2g.faces ); + buildL2GMappings( keys( graph.c2f ), g2l.cells, l2g.cells ); - return result; + return { g2l, l2g }; } template< class GI, class LI > @@ -282,25 +282,23 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, return CellBlkImpl( intConv< localIndex >( numCells ), ghostRank_, convertToA2d( c2n, 8 ), convertToA2d( c2e, 12 ), convertToA2d( c2f, 6 ), std::move( l2g ) ); } -DownwardMappings buildDownwardMappings( GlobalNumberings const & numberings, - MeshGraph const & graph, - GhostRecv const & recv, - GhostSend const & send ) +DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, + MeshGraph const & graph ) { DownwardMappings res; // Building the `e2n` (edges to nodes) mapping for( auto const & [egi, ngis]: graph.e2n ) { - NodeLocIdx const nli0 = numberings.ng2l.at( std::get< 0 >( ngis ) ); - NodeLocIdx const nli1 = numberings.ng2l.at( std::get< 1 >( ngis ) ); - res.e2n[numberings.eg2l.at( egi )] = { nli0, nli1 }; + NodeLocIdx const nli0 = g2l.nodes.at( std::get< 0 >( ngis ) ); + NodeLocIdx const nli1 = g2l.nodes.at( std::get< 1 >( ngis ) ); + res.e2n[g2l.edges.at( egi )] = { nli0, nli1 }; } // Building the `f2n` (face to nodes) and `f2e` (faces to edges) mappings for( auto const & [fgi, edgeInfos]: graph.f2e ) { - FaceLocIdx const & fli = numberings.fg2l.at( fgi ); + FaceLocIdx const & fli = g2l.faces.at( fgi ); std::vector< NodeLocIdx > & nodes = res.f2n[fli]; std::vector< EdgeLocIdx > & edges = res.f2e[fli]; @@ -311,16 +309,16 @@ DownwardMappings buildDownwardMappings( GlobalNumberings const & numberings, for( EdgeInfo const & edgeInfo: edgeInfos ) { std::tuple< NodeGlbIdx, NodeGlbIdx > const & ngis = graph.e2n.at( edgeInfo.index ); - nodes.emplace_back( edgeInfo.start == 0 ? numberings.ng2l.at( std::get< 0 >( ngis ) ) : numberings.ng2l.at( std::get< 1 >( ngis ) ) ); + nodes.emplace_back( edgeInfo.start == 0 ? g2l.nodes.at( std::get< 0 >( ngis ) ) : g2l.nodes.at( std::get< 1 >( ngis ) ) ); - edges.emplace_back( numberings.eg2l.at( edgeInfo.index ) ); + edges.emplace_back( g2l.edges.at( edgeInfo.index ) ); } } // Building the `c2n` (cell to nodes), `c2e` (cell to edges) and `c2f` (cell to faces) mappings for( auto const & [cgi, faceInfos]: graph.c2f ) { - CellLocIdx const & cli = numberings.cg2l.at( cgi ); + CellLocIdx const & cli = g2l.cells.at( cgi ); std::vector< FaceLocIdx > & faces = res.c2f[cli]; std::vector< EdgeLocIdx > & edges = res.c2e[cli]; @@ -331,7 +329,7 @@ DownwardMappings buildDownwardMappings( GlobalNumberings const & numberings, // c2f for( FaceInfo const & faceInfo: faceInfos ) { - faces.emplace_back( numberings.fg2l.at( faceInfo.index ) ); + faces.emplace_back( g2l.faces.at( faceInfo.index ) ); } // c2e @@ -347,8 +345,8 @@ DownwardMappings buildDownwardMappings( GlobalNumberings const & numberings, FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type. FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7) - std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( numberings.fg2l.at( bottomFace.index ) ); - std::vector< NodeLocIdx > const & topNodes = res.f2n.at( numberings.fg2l.at( topFace.index ) ); + std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( g2l.faces.at( bottomFace.index ) ); + std::vector< NodeLocIdx > const & topNodes = res.f2n.at( g2l.faces.at( topFace.index ) ); std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); @@ -425,7 +423,7 @@ void buildPods( MeshGraph const & owned, { MeshGraph const graph = mergeMeshGraph( owned, present, ghosts ); - GlobalNumberings const numberings = buildL2GMappings( graph ); + auto const [g2l, l2g] = buildL2GMappings( graph ); // GEOS_LOG_RANK( "numberings ng2l = " << json( numberings.ng2l ) ); // GEOS_LOG_RANK( "owned mesh graph nodes = " << json( owned.n ) ); // GEOS_LOG_RANK( "present mesh graph nodes = " << json( present.n ) ); @@ -437,39 +435,39 @@ void buildPods( MeshGraph const & owned, // MpiWrapper::barrier(); - DownwardMappings const downwardMappings = buildDownwardMappings( numberings, graph, recv, send ); + DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); - NodeMgrImpl const nodeMgr = makeFlavorlessNodeMgrImpl( std::size( numberings.ng2l ), - buildGhostRank( numberings.ng2l, send.nodes, recv.nodes ), + NodeMgrImpl const nodeMgr = makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), + buildGhostRank( g2l.nodes, send.nodes, recv.nodes ), upwardMappings.n2e, upwardMappings.n2f, upwardMappings.n2c, - numberings.ng2l, - numberings.nl2g ); + g2l.nodes, + l2g.nodes ); - EdgeMgrImpl const edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( numberings.eg2l ), - buildGhostRank( numberings.eg2l, send.edges, recv.edges ), + EdgeMgrImpl const edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), + buildGhostRank( g2l.edges, send.edges, recv.edges ), downwardMappings.e2n, upwardMappings.e2f, - numberings.eg2l, - numberings.el2g ); + g2l.edges, + l2g.edges ); - FaceMgrImpl const faceMgr = makeFlavorlessFaceMgrImpl( std::size( numberings.fg2l ), - buildGhostRank( numberings.fg2l, send.faces, recv.faces ), + FaceMgrImpl const faceMgr = makeFlavorlessFaceMgrImpl( std::size( g2l.faces ), + buildGhostRank( g2l.faces, send.faces, recv.faces ), downwardMappings.f2n, downwardMappings.f2e, upwardMappings.f2c, - numberings.fg2l, - numberings.fl2g ); - - CellBlkImpl const cellBlock = makeFlavorlessCellBlkImpl( std::size( numberings.cg2l ), - buildGhostRank( numberings.cg2l, send.cells, recv.cells ), - downwardMappings.c2n, - downwardMappings.c2e, - downwardMappings.c2f, - numberings.cg2l, - numberings.cl2g ); + g2l.faces, + l2g.faces ); + + CellBlkImpl const cellBlock = makeFlavorlessCellBlkImpl( std::size( g2l.cells ), + buildGhostRank( g2l.cells, send.cells, recv.cells ), + downwardMappings.c2n, + downwardMappings.c2e, + downwardMappings.c2f, + g2l.cells, + l2g.cells ); } } From 93f51691ab21e6c5ad864b2b337a9c39182acc6e Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:38:23 -0700 Subject: [PATCH 086/106] Split the construction of the upward matrix information from the ghosting algorithm. --- .../mesh/generators/NewGhosting.cpp | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index edc93af5ff1..c4599c00b39 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -457,37 +457,38 @@ std::array< std::size_t, N > decode( std::size_t const & basis, } -std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph const & owned, - MeshGraph const & present, - MaxGlbIdcs const & gis, - MpiRank curRank ) +struct Adjacency { - FindGeometricalType const convert( gis.nodes, gis.edges, gis.faces, gis.cells ); - using Geom = FindGeometricalType::Geom; + std::vector< int > ownedGlbIdcs; // TODO Use some strongly typed ints. + std::vector< int > otherGlbIdcs; // TODO Use some strongly typed ints. + std::vector< int > numEntriesPerRow; + std::vector< std::vector< int > > indices; + std::vector< std::vector< double > > values; +}; - std::size_t const n = convert.numEntries(); // Total number of entries in the graph. - std::size_t const numOwnedNodes = std::size( owned.n ); - std::size_t const numOwnedEdges = std::size( owned.e2n ); - std::size_t const numOwnedFaces = std::size( owned.f2e ); - std::size_t const numOwnedCells = std::size( owned.c2f ); - std::size_t const numOwned = numOwnedNodes + numOwnedEdges + numOwnedFaces + numOwnedCells; +Adjacency buildAdjacency( MeshGraph const & owned, + MeshGraph const & present, + FindGeometricalType const & convert ) +{ + Adjacency adjacency; - std::size_t const numOtherNodes = std::size( present.n ); - std::size_t const numOtherEdges = std::size( present.e2n ); - std::size_t const numOtherFaces = std::size( present.f2e ); - std::size_t const numOther = numOtherNodes + numOtherEdges + numOtherFaces; + std::size_t const numOwned = std::size( owned.n ) + std::size( owned.e2n ) + std::size( owned.f2e ) + std::size( owned.c2f ); + std::size_t const numOther = std::size( present.n ) + std::size( present.e2n ) + std::size( present.f2e ); + + // Aliases + std::vector< int > & ownedGlbIdcs = adjacency.ownedGlbIdcs; + std::vector< int > & otherGlbIdcs = adjacency.otherGlbIdcs; + std::vector< int > & numEntriesPerRow = adjacency.numEntriesPerRow; + std::vector< std::vector< int > > & indices = adjacency.indices; + std::vector< std::vector< double > > & values = adjacency.values; - std::vector< int > ownedGlbIdcs, numEntriesPerRow; // TODO I couldn't use a vector of `std::size_t` - std::vector< std::vector< int > > indices; - std::vector< std::vector< double > > values; ownedGlbIdcs.reserve( numOwned ); + otherGlbIdcs.reserve( numOther ); numEntriesPerRow.reserve( numOwned ); indices.reserve( numOwned ); - indices.reserve( numOwned ); + values.reserve( numOwned ); - std::vector< int > otherGlbIdcs; // TODO I couldn't use a vector of `std::size_t` - otherGlbIdcs.reserve( numOther ); for( NodeGlbIdx const & ngi: present.n ) { otherGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); @@ -593,6 +594,30 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph GEOS_ASSERT_EQ( indices[i].size(), std::size_t( numEntriesPerRow[i] ) ); } + return adjacency; +} + +std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & owned, + MeshGraph const & present, + MaxGlbIdcs const & gis, + MpiRank curRank ) +{ + FindGeometricalType const convert( gis.nodes, gis.edges, gis.faces, gis.cells ); + using Geom = FindGeometricalType::Geom; + + std::size_t const n = convert.numEntries(); // Total number of entries in the graph. + + Adjacency const adjacency = buildAdjacency( owned, present, convert ); + std::size_t const numOwned = std::size( adjacency.ownedGlbIdcs ); + std::size_t const numOther = std::size( adjacency.otherGlbIdcs ); + + // Aliases + std::vector< int > const & ownedGlbIdcs = adjacency.ownedGlbIdcs; + std::vector< int > const & otherGlbIdcs = adjacency.otherGlbIdcs; + std::vector< int > const & numEntriesPerRow = adjacency.numEntriesPerRow; + std::vector< std::vector< int > > const & indices = adjacency.indices; + std::vector< std::vector< double > > const & values = adjacency.values; + Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); Epetra_Map const ownedMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); @@ -808,7 +833,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > assembleAdjacencyMatrix( MeshGraph // - the number of columns being the number of nodes in the mesh graph, // - the number of rows being the total number graph nodes that's missing on ranks. // - all its terms will be `1`. - // Combined with the adjacency matrix which conveya a lot of connections information, + // Combined with the adjacency matrix which conveys a lot of connections information, // we'll be able to create the final `missingMappings`. // We also create `maps` with specific MPI ranks ownerships and offsets, // so we'll be able to retrieve the information on the ranks that need it (nothing more, nothing less). @@ -993,7 +1018,7 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v // } // MpiWrapper::barrier(); - auto const [ghosts, recv, send] = assembleAdjacencyMatrix( owned, present, matrixOffsets, curRank ); + auto const [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); buildPods( owned, present, ghosts, recv, send ); From b3b1b5f80a3722dbd4b50057451185c537071fd2 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 4 Jun 2024 17:05:10 -0700 Subject: [PATCH 087/106] Use one single pair of vectors when copying the rows. --- .../mesh/generators/NewGhosting.cpp | 79 ++++++++++--------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index c4599c00b39..d00471b553a 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -724,14 +724,19 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostInfo.mat", ghostingFootprint ); - int extracted2 = 0; - std::vector< double > extractedValues2( commSize ); // TODO improve with resize... - std::vector< int > extractedIndices2( commSize ); + int extracted = 0; + std::vector< double > extractedValues; + std::vector< int > extractedIndices; GhostSend send; for( int const & index: ownedGlbIdcs ) { - ghostingFootprint.ExtractGlobalRowCopy( index, commSize, extracted2, extractedValues2.data(), extractedIndices2.data() ); + int const length = ghostingFootprint.NumGlobalEntries( index ); + extractedValues.resize( length ); + extractedIndices.resize( length ); + ghostingFootprint.ExtractGlobalRowCopy( index, length, extracted, extractedValues.data(), extractedIndices.data() ); + GEOS_ASSERT_EQ( extracted, length ); + std::set< MpiRank > * sendingTo = nullptr; switch( convert.getGeometricalType( index ) ) { @@ -755,11 +760,15 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & sendingTo = &send.cells[convert.toCellGlbIdx( index )]; break; } + default: + { + GEOS_ERROR( "Internal error" ); + } } - for( int ii = 0; ii < extracted2; ++ii ) + for( int ii = 0; ii < extracted; ++ii ) { - MpiRank const rank{ extractedIndices2[ii] }; + MpiRank const rank{ extractedIndices[ii] }; if( rank != curRank ) { sendingTo->insert( rank ); @@ -767,12 +776,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } } - int extracted = 0; - int const length = ghostExchange.NumGlobalEntries( curRank.get() ); - std::vector< double > extractedValues( length ); - std::vector< int > extractedIndices( length ); - ghostExchange.ExtractGlobalRowCopy( curRank.get(), length, extracted, extractedValues.data(), extractedIndices.data() ); - GEOS_ASSERT_EQ( extracted, length ); + int const numNeededIndices = ghostExchange.NumGlobalEntries( curRank.get() ); + extractedValues.resize( numNeededIndices ); + extractedIndices.resize( numNeededIndices ); + ghostExchange.ExtractGlobalRowCopy( curRank.get(), numNeededIndices, extracted, extractedValues.data(), extractedIndices.data() ); + GEOS_ASSERT_EQ( extracted, numNeededIndices ); std::set< int > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); std::set< int > receivedIndices; // The graph nodes that my neighbors will send me. @@ -783,6 +791,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), std::back_inserter( notPresentIndices ) ); + GEOS_ASSERT_EQ( intConv< int >( std::size( allNeededIndices ) ), numNeededIndices ); GhostRecv recv; for( int i = 0; i < extracted; ++i ) @@ -862,17 +871,13 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & MeshGraph ghosts; - int ext = 0; - std::vector< double > extValues; - std::vector< int > extIndices; - for( int i = 0; i < int( numMissingIndices ); ++i ) { - int const length2 = missingMappings.NumGlobalEntries( offset + i ); - extValues.resize( length2 ); - extIndices.resize( length2 ); - missingMappings.ExtractGlobalRowCopy( offset + i, length2, ext, extValues.data(), extIndices.data() ); - GEOS_ASSERT_EQ( ext, length2 ); + int const length = missingMappings.NumGlobalEntries( offset + i ); + extractedValues.resize( length ); + extractedIndices.resize( length ); + missingMappings.ExtractGlobalRowCopy( offset + i, length, extracted, extractedValues.data(), extractedIndices.data() ); + GEOS_ASSERT_EQ( extracted, length ); int const index = notPresentIndices[i]; Geom const geometricalType = convert.getGeometricalType( index ); if( geometricalType == Geom::NODE ) @@ -880,16 +885,16 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // The case of nodes is a bit different from the other cases // because nodes do not rely on other geometrical quantities. // We simply have to extract and store their own index. - GEOS_ASSERT_EQ( length2, 1 ); - GEOS_ASSERT_EQ( extIndices[0], int(extValues[0]) ); - ghosts.n.insert( convert.toNodeGlbIdx( extIndices[0] ) ); + GEOS_ASSERT_EQ( length, 1 ); + GEOS_ASSERT_EQ( extractedIndices[0], int(extractedValues[0]) ); + ghosts.n.insert( convert.toNodeGlbIdx( extractedIndices[0] ) ); continue; } - auto const cit = std::find( std::cbegin( extIndices ), std::cend( extIndices ), index ); - std::ptrdiff_t const numGeomQuantitiesIdx = std::distance( std::cbegin( extIndices ), cit ); - int const numGeomQuantities = int( extValues[numGeomQuantitiesIdx] ); - GEOS_ASSERT_EQ( ext, numGeomQuantities + 1 ); + auto const cit = std::find( std::cbegin( extractedIndices ), std::cend( extractedIndices ), index ); + std::ptrdiff_t const numGeomQuantitiesIdx = std::distance( std::cbegin( extractedIndices ), cit ); + int const numGeomQuantities = int( extractedValues[numGeomQuantitiesIdx] ); + GEOS_ASSERT_EQ( extracted, numGeomQuantities + 1 ); switch( geometricalType ) { @@ -899,15 +904,15 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & GEOS_ASSERT_EQ( numNodes, 2 ); std::array< NodeGlbIdx, 2 > order{}; - for( int ii = 0; ii < ext; ++ii ) + for( int ii = 0; ii < extracted; ++ii ) { if( ii == numGeomQuantitiesIdx ) { continue; } - NodeGlbIdx const ngi = convert.toNodeGlbIdx( extIndices[ii] ); - integer const ord = integer( extValues[ii] - 1 ); + NodeGlbIdx const ngi = convert.toNodeGlbIdx( extractedIndices[ii] ); + integer const ord = integer( extractedValues[ii] - 1 ); GEOS_ASSERT( ord == 0 or ord == 1 ); order[ord] = ngi; } @@ -922,15 +927,15 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & { int const & numEdges = numGeomQuantities; // Alias std::map< integer, EdgeInfo > order; - for( int ii = 0; ii < ext; ++ii ) + for( int ii = 0; ii < extracted; ++ii ) { if( ii == numGeomQuantitiesIdx ) { continue; } - EdgeGlbIdx const egi = convert.toEdgeGlbIdx( extIndices[ii] ); - std::array< std::size_t, 2 > const decoded = decode< 2 >( numEdges, std::size_t( extValues[ii] - 1 ) ); + EdgeGlbIdx const egi = convert.toEdgeGlbIdx( extractedIndices[ii] ); + std::array< std::size_t, 2 > const decoded = decode< 2 >( numEdges, std::size_t( extractedValues[ii] - 1 ) ); order[decoded[1]] = { egi, intConv< std::uint8_t >( decoded[0] ) }; GEOS_ASSERT( decoded[0] == 0 or decoded[0] == 1 ); } @@ -949,15 +954,15 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & { int const & numFaces = numGeomQuantities; // Alias // TODO This should receive the cell type instead. std::map< integer, FaceInfo > order; - for( int ii = 0; ii < ext; ++ii ) + for( int ii = 0; ii < extracted; ++ii ) { if( ii == numGeomQuantitiesIdx ) { continue; } - FaceGlbIdx const fgi = convert.toFaceGlbIdx( extIndices[ii] ); - std::array< std::size_t, 3 > const decoded = decode< 3 >( numFaces, std::size_t( extValues[ii] - 1 ) ); + FaceGlbIdx const fgi = convert.toFaceGlbIdx( extractedIndices[ii] ); + std::array< std::size_t, 3 > const decoded = decode< 3 >( numFaces, std::size_t( extractedValues[ii] - 1 ) ); order[decoded[2]] = { fgi, intConv< bool >( decoded[0] ), intConv< std::uint8_t >( decoded[1] ) }; } GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numFaces ) ); From 29aaf4eab69d7f470d9ec85348d6c4c8d2bcf669 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:32:34 -0700 Subject: [PATCH 088/106] Node positions are now exchanged. --- .../mesh/generators/BuildPods.cpp | 4 +- .../mesh/generators/BuildPods.hpp | 3 +- .../mesh/generators/NewGhosting.cpp | 205 ++++++++++++++---- 3 files changed, 161 insertions(+), 51 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 2098ae2d288..1def31e752c 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -72,7 +72,7 @@ std::tuple< GlobalToLocal, LocalToGlobal > buildL2GMappings( MeshGraph const & g return mapKeys< std::set >( m ); }; - buildL2GMappings( graph.n, g2l.nodes, l2g.nodes ); + buildL2GMappings( keys( graph.n2pos ), g2l.nodes, l2g.nodes ); buildL2GMappings( keys( graph.e2n ), g2l.edges, l2g.edges ); buildL2GMappings( keys( graph.f2e ), g2l.faces, l2g.faces ); buildL2GMappings( keys( graph.c2f ), g2l.cells, l2g.cells ); @@ -110,7 +110,7 @@ MeshGraph mergeMeshGraph( MeshGraph const & owned, result.c2f.insert( std::cbegin( graph.c2f ), std::cend( graph.c2f ) ); result.f2e.insert( std::cbegin( graph.f2e ), std::cend( graph.f2e ) ); result.e2n.insert( std::cbegin( graph.e2n ), std::cend( graph.e2n ) ); - result.n.insert( std::cbegin( graph.n ), std::cend( graph.n ) ); + result.n2pos.insert( std::cbegin( graph.n2pos ), std::cend( graph.n2pos ) ); } return result; } diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index d9af664434f..60da63a5288 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -17,6 +17,7 @@ #include "Indices.hpp" +#include #include #include @@ -62,7 +63,7 @@ struct MeshGraph std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; std::map< FaceGlbIdx, std::vector< EdgeInfo > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? - std::set< NodeGlbIdx > n; + std::map< NodeGlbIdx, std::array< double, 3 > > n2pos; }; struct GhostSend diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index d00471b553a..437dfdd2fc0 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -138,7 +138,22 @@ void to_json( json & j, j = json{ { "c2f", v.f2e }, { "f2e", v.f2e }, { "e2n", v.e2n }, - { "n", v.n } }; + { "n2pos", v.n2pos } }; +} + + +std::map< NodeGlbIdx, vtkIdType > buildNgiToVtk( vtkSmartPointer< vtkDataSet > mesh ) +{ + std::map< NodeGlbIdx, vtkIdType > res; + + vtkIdTypeArray const * globalPtIds = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); + + for( vtkIdType i = 0; i < mesh->GetNumberOfPoints(); ++i ) + { + res[NodeGlbIdx{ globalPtIds->GetValue( i ) }] = i; + } + + return res; } /** @@ -162,10 +177,15 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > return curRank == *std::min_element( std::cbegin( ranks ), std::cend( ranks ) ); }; + std::map< NodeGlbIdx, vtkIdType > const n2vtk = buildNgiToVtk( mesh ); for( auto const & [ranks, nodes]: buckets.nodes ) { - std::set< NodeGlbIdx > & n = isCurrentRankOwning( ranks ) ? owned.n : present.n; - n.insert( std::cbegin( nodes ), std::cend( nodes ) ); + std::map< NodeGlbIdx, std::array< double, 3 > > & n2pos = isCurrentRankOwning( ranks ) ? owned.n2pos : present.n2pos; + for( NodeGlbIdx const & ngi: nodes ) + { + double const * pos = mesh->GetPoint( n2vtk.at( ngi ) ); + n2pos[ngi] = { pos[0], pos[1], pos[2] }; + } } for( auto const & [ranks, edges]: buckets.edges ) @@ -309,6 +329,7 @@ Epetra_CrsMatrix multiply( int commSize, // Downward (c -> f -> e -> n) auto tDownward = makeTranspose( upward ); // TODO check the algorithm to understand what's more relevant. + // TODO why do we have to perform the transposition ourselves instead of using the flag from `EpetraExt::MatrixMatrix::Multiply`. Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_u0_2, false, result_d0_0, false ); @@ -456,9 +477,9 @@ std::array< std::size_t, N > decode( std::size_t const & basis, return result; } - struct Adjacency { + std::vector< int > ownedNodesIdcs; // TODO Use some strongly typed ints. std::vector< int > ownedGlbIdcs; // TODO Use some strongly typed ints. std::vector< int > otherGlbIdcs; // TODO Use some strongly typed ints. std::vector< int > numEntriesPerRow; @@ -466,30 +487,31 @@ struct Adjacency std::vector< std::vector< double > > values; }; - Adjacency buildAdjacency( MeshGraph const & owned, MeshGraph const & present, FindGeometricalType const & convert ) { Adjacency adjacency; - std::size_t const numOwned = std::size( owned.n ) + std::size( owned.e2n ) + std::size( owned.f2e ) + std::size( owned.c2f ); - std::size_t const numOther = std::size( present.n ) + std::size( present.e2n ) + std::size( present.f2e ); + std::size_t const numOwned = std::size( owned.n2pos ) + std::size( owned.e2n ) + std::size( owned.f2e ) + std::size( owned.c2f ); + std::size_t const numOther = std::size( present.n2pos ) + std::size( present.e2n ) + std::size( present.f2e ); // Aliases + std::vector< int > & ownedNodesIdcs = adjacency.ownedNodesIdcs; std::vector< int > & ownedGlbIdcs = adjacency.ownedGlbIdcs; std::vector< int > & otherGlbIdcs = adjacency.otherGlbIdcs; std::vector< int > & numEntriesPerRow = adjacency.numEntriesPerRow; std::vector< std::vector< int > > & indices = adjacency.indices; std::vector< std::vector< double > > & values = adjacency.values; + ownedNodesIdcs.reserve( std::size( owned.n2pos ) ); ownedGlbIdcs.reserve( numOwned ); otherGlbIdcs.reserve( numOther ); numEntriesPerRow.reserve( numOwned ); indices.reserve( numOwned ); values.reserve( numOwned ); - for( NodeGlbIdx const & ngi: present.n ) + for( auto const & [ngi, _]: present.n2pos ) { otherGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); } @@ -504,12 +526,14 @@ Adjacency buildAdjacency( MeshGraph const & owned, std::sort( std::begin( otherGlbIdcs ), std::end( otherGlbIdcs ) ); GEOS_ASSERT_EQ( numOther, std::size( otherGlbIdcs ) ); - for( NodeGlbIdx const & ngi: owned.n ) + for( auto const & [ngi, _]: owned.n2pos ) { // Nodes depend on no other geometrical entity, // so we only have one entry `1` in the diagonal of the matrix, // because we add the identity to the adjacency matrix. - ownedGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); + int const i = convert.fromNodeGlbIdx( ngi ); + ownedNodesIdcs.emplace_back( i ); + ownedGlbIdcs.emplace_back( i ); numEntriesPerRow.emplace_back( 0 + 1 ); // `+1` comes from the diagonal indices.emplace_back( 1, ownedGlbIdcs.back() ); values.emplace_back( 1, ownedGlbIdcs.back() ); @@ -612,6 +636,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::size_t const numOther = std::size( adjacency.otherGlbIdcs ); // Aliases + std::vector< int > const & ownedNodesIdcs = adjacency.ownedNodesIdcs; std::vector< int > const & ownedGlbIdcs = adjacency.ownedGlbIdcs; std::vector< int > const & otherGlbIdcs = adjacency.otherGlbIdcs; std::vector< int > const & numEntriesPerRow = adjacency.numEntriesPerRow; @@ -791,6 +816,12 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), std::back_inserter( notPresentIndices ) ); + std::vector< int > notPresentNodes; + std::copy_if( std::cbegin( notPresentIndices ), std::cend( notPresentIndices ), + std::back_inserter( notPresentNodes ), [&]( int const & i ) + { + return convert.getGeometricalType( i ) == Geom::NODE; + } ); GEOS_ASSERT_EQ( intConv< int >( std::size( allNeededIndices ) ), numNeededIndices ); GhostRecv recv; @@ -825,6 +856,10 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & recv.cells[convert.toCellGlbIdx( index )] = sender; break; } + default: + { + GEOS_ERROR( "Internal error" ); + } } } @@ -835,59 +870,128 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // } // At this point, each rank knows what it has to send to and what it is going to receive from the other ranks. - // The remaining action is about retrieving the additional graph information - // for the new geometrical quantities that will be sent by the neighbors. // - // In order to do that, we build the `missing` matrix, which is rectangular, - // - the number of columns being the number of nodes in the mesh graph, - // - the number of rows being the total number graph nodes that's missing on ranks. - // - all its terms will be `1`. - // Combined with the adjacency matrix which conveys a lot of connections information, - // we'll be able to create the final `missingMappings`. - // We also create `maps` with specific MPI ranks ownerships and offsets, - // so we'll be able to retrieve the information on the ranks that need it (nothing more, nothing less). - std::size_t const numMissingIndices = std::size( notPresentIndices ); - std::size_t const numGlobalMissingIndices = MpiWrapper::sum( numMissingIndices ); // A missing node can be missing on multiple ranks and that's OK. - -// Epetra_Map const missingIndicesMap( intConv< long long int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); - Epetra_Map const missingIndicesMap( intConv< int >( numGlobalMissingIndices ), intConv< int >( numMissingIndices ), 0, comm ); - -// std::size_t const offset = MpiWrapper::prefixSum< std::size_t >( numMissingIndices ); - int const offset = *missingIndicesMap.MyGlobalElements(); // Needed to compute the global row -// GEOS_LOG_RANK( "numMissingIndices, numGlobalMissingIndices, offset = " << numMissingIndices << ", " << numGlobalMissingIndices << ", " << offset ); - - Epetra_CrsMatrix missing( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); - for( std::size_t i = 0; i < numMissingIndices; ++i ) + // The remaining action is about + // - retrieving the additional graph information + // for the new geometrical quantities that will be sent by the neighbors. + // - retrieving the positions of the ghosted nodes: + // knowing the index of the nodes is not enough. + // + // In order to do that, we build the `missingIndicator` matrix, which is rectangular: + // - The number of columns is the number of graph nodes in the mesh graph. + // - The number of rows is the total number of graph nodes that are missing on ranks. + // Note that the same quantity can be missing on multiple ranks, and that's handled. + // - The non-zero terms equal `1`, meaning that a given quantity (column) is missing on a given rank (row). + // + // Combining `missingIndicator` with the adjacency matrix which conveys a lot of connections information, + // we'll be able to create the final `missingIndices` matrix, + // with `range` and `domain` maps (MPI ranks ownerships and offsets) appropriately defined + // such that the rows will be available to any ranks that need them (nothing more, nothing less). + // + // To get the `missingNodePos` matrix which will convey the ghosted nodes positions that are missing on the rank, + // we first create a specific `missingNodesIndicator` matrix, which is alike `missingIndicator` but only for the nodes. + // We could have used `missingIndicator` and ditched the superfluous information, + // but the node positions are reals, where the connections are integers. + // As long as we're using `Epetra`, where the type of matrix term is `double`, this makes no critical difference. + // But when we switch to `Tpetra`, where we can select the type of the matrix term (and therefore use integers where we need integers), + // this may have become an issue. + // So the work has been done to separate `missingIndicator` and `missingNodesIndicator`. + // + // `missingNodesIndicator` is a rectangular like `missingIndicator`: + // - The number of columns is the number of graph nodes in the mesh graph + // that actually are physical nodes in the mesh. + // - The number of rows is the total number of graph nodes (that actually are physical nodes in the mesh) that are missing on ranks. + // - The non-zero terms equal `1`, meaning that a given quantity (column) is missing on a given rank (row). + std::size_t const numLocMissingCompound[2] = { std::size( notPresentIndices ), std::size( notPresentNodes ) }; + std::size_t numGlbMissingCompound[2] = { 0, 0 }; + MpiWrapper::allReduce( numLocMissingCompound, numGlbMissingCompound, 2, MPI_SUM ); + std::size_t const & numLocMissingIndices = numLocMissingCompound[0]; + std::size_t const & numLocMissingNodes = numLocMissingCompound[1]; + std::size_t const & numGlbMissingIndices = numGlbMissingCompound[0]; + std::size_t const & numGlbMissingNodes = numGlbMissingCompound[1]; + + std::size_t const numGlbNodes = gis.nodes.get() + 1; + std::size_t const numOwnedNodes = std::size( ownedNodesIdcs ); + + Epetra_Map const missingIndicesMap( intConv< int >( numGlbMissingIndices ), intConv< int >( numLocMissingIndices ), 0, comm ); + Epetra_Map const missingNodesMap( intConv< int >( numGlbMissingNodes ), intConv< int >( numLocMissingNodes ), 0, comm ); + + // Following information is needed to compute the global row. + int const missingIndicesOffset = *missingIndicesMap.MyGlobalElements(); + int const missingNodesOffset = *missingNodesMap.MyGlobalElements(); + + // Indicator matrix for the all the missing quantities (nodes, edges, faces and cells). + Epetra_CrsMatrix missingIndicator( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); + for( std::size_t i = 0; i < numLocMissingIndices; ++i ) + { + missingIndicator.InsertGlobalValues( missingIndicesOffset + i, 1, ones.data(), ¬PresentIndices[i] ); + } + missingIndicator.FillComplete( ownedMap, missingIndicesMap ); + + // Indicator matrix only for the nodes. + Epetra_CrsMatrix missingNodesIndicator( Epetra_DataAccess::Copy, missingNodesMap, 1, true ); + for( int i = 0; i < intConv< int >( numLocMissingNodes ); ++i ) { - missing.InsertGlobalValues( offset + i, 1, ones.data(), ¬PresentIndices[i] ); + missingNodesIndicator.InsertGlobalValues( missingNodesOffset + i, 1, ones.data(), ¬PresentNodes[i] ); } - missing.FillComplete( ownedMap, missingIndicesMap ); + Epetra_Map const ownedNodesMap( numGlbNodes, numOwnedNodes, ownedNodesIdcs.data(), 0, comm ); + missingNodesIndicator.FillComplete( ownedNodesMap, missingNodesMap ); + + // The `nodePositions` matrix is rectangular. + // - Its number of rows is the total number of nodes in the mesh. + // - Its number of columns is 3: the x, y, and z coordinates of the nodes. + Epetra_CrsMatrix nodePositions( Epetra_DataAccess::Copy, ownedNodesMap, 3, true ); + std::vector< int > const zot{ 0, 1, 2 }; // zot: zero, one, two. + for( auto const & [ngi, pos]: owned.n2pos ) + { + nodePositions.InsertGlobalValues( convert.fromNodeGlbIdx( ngi ), 3, pos.data(), zot.data() ); + } + Epetra_Map const threeMap = Epetra_Map( 3, 0, comm ); + nodePositions.FillComplete( threeMap, ownedNodesMap ); + // `missingIndices` will contain the missing connectivity information. auto tDownward = makeTranspose( upward ); // TODO give it to multiply! - Epetra_CrsMatrix missingMappings( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); - EpetraExt::MatrixMatrix::Multiply( missing, false, *tDownward, true, missingMappings, false ); - missingMappings.FillComplete( ownedMap, missingIndicesMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingMappings ); + Epetra_CrsMatrix missingIndices( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); + EpetraExt::MatrixMatrix::Multiply( missingIndicator, false, *tDownward, true, missingIndices, false ); + missingIndices.FillComplete( ownedMap, missingIndicesMap ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingIndices ); + + // `missingNodePos` will contain the missing node positions. + Epetra_CrsMatrix missingNodePos( Epetra_DataAccess::Copy, missingNodesMap, 1, false ); + EpetraExt::MatrixMatrix::Multiply( missingNodesIndicator, false, nodePositions, false, missingNodePos, false ); + missingNodePos.FillComplete( threeMap, missingNodesMap ); MeshGraph ghosts; - for( int i = 0; i < int( numMissingIndices ); ++i ) + for( int i = 0; i < int( numLocMissingIndices ); ++i ) { - int const length = missingMappings.NumGlobalEntries( offset + i ); + int const index = notPresentIndices[i]; + Geom const geometricalType = convert.getGeometricalType( index ); + + int const length = missingIndices.NumGlobalEntries( missingIndicesOffset + i ); extractedValues.resize( length ); extractedIndices.resize( length ); - missingMappings.ExtractGlobalRowCopy( offset + i, length, extracted, extractedValues.data(), extractedIndices.data() ); + missingIndices.ExtractGlobalRowCopy( missingIndicesOffset + i, length, extracted, extractedValues.data(), extractedIndices.data() ); GEOS_ASSERT_EQ( extracted, length ); - int const index = notPresentIndices[i]; - Geom const geometricalType = convert.getGeometricalType( index ); if( geometricalType == Geom::NODE ) { - // The case of nodes is a bit different from the other cases - // because nodes do not rely on other geometrical quantities. - // We simply have to extract and store their own index. + // The case of nodes is a bit different from the other cases, + // because nodes do not rely on other geometrical quantities, + // but we need to extract the position of the node instead. + // In order to extract these positions, we use the other matrix `missingNodePos`. GEOS_ASSERT_EQ( length, 1 ); - GEOS_ASSERT_EQ( extractedIndices[0], int(extractedValues[0]) ); - ghosts.n.insert( convert.toNodeGlbIdx( extractedIndices[0] ) ); + int const lengthPos = missingNodePos.NumGlobalEntries( missingNodesOffset + i ); + extractedValues.resize( lengthPos ); + extractedIndices.resize( lengthPos ); + missingNodePos.ExtractGlobalRowCopy( missingNodesOffset + i, lengthPos, extracted, extractedValues.data(), extractedIndices.data() ); + GEOS_ASSERT_EQ( extracted, lengthPos ); + GEOS_ASSERT_EQ( lengthPos, 3 ); + GEOS_ASSERT_EQ( index, notPresentNodes[i] ); + std::array< double, 3 > & pos = ghosts.n2pos[convert.toNodeGlbIdx( index )]; + for( auto dim = 0; dim < 3; ++dim ) + { + pos[extractedIndices[dim]] = extractedValues[dim]; + } continue; } @@ -998,6 +1102,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // } // GEOS_LOG_RANK( "my final neighbors are " << json( ownerships.neighbors ) ); +// if( curRank == 1_mpi ) +// { +// GEOS_LOG_RANK( "ghosts_n2ps = " << json( ghosts.n2pos ) ); +// } + return { std::move( ghosts ), std::move( recv ), std::move( send ) }; } From a799687ef1f66926e4666a81f1f20ea4cf943ad3 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:57:31 -0700 Subject: [PATCH 089/106] Give the node positions to the NodeMgr. --- .../mesh/generators/BuildPods.cpp | 41 ++++++++++++++++++- src/coreComponents/mesh/generators/Pods.cpp | 4 +- src/coreComponents/mesh/generators/Pods.hpp | 2 + 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 1def31e752c..4c720b937ff 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -202,6 +202,12 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::map< EdgeGlbIdx, EdgeLocIdx > const & eg2l, std::map< EdgeLocIdx, EdgeGlbIdx > const & el2g ) { + GEOS_ASSERT_EQ( numEdges, std::size( ghostRank ) ); + GEOS_ASSERT_EQ( numEdges, std::size( e2n ) ); + GEOS_ASSERT_EQ( numEdges, std::size( e2f ) ); + GEOS_ASSERT_EQ( numEdges, std::size( eg2l ) ); + GEOS_ASSERT_EQ( numEdges, std::size( el2g ) ); + array2d< localIndex > e2n_( numEdges, 2 ); for( int i = 0; i < intConv< int >( numEdges ); ++i ) { @@ -229,6 +235,13 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, std::map< FaceGlbIdx, FaceLocIdx > const & fg2l, std::map< FaceLocIdx, FaceGlbIdx > const & fl2g ) { + GEOS_ASSERT_EQ( numFaces, std::size( ghostRank ) ); + GEOS_ASSERT_EQ( numFaces, std::size( f2n ) ); + GEOS_ASSERT_EQ( numFaces, std::size( f2e ) ); + GEOS_ASSERT_EQ( numFaces, std::size( f2c ) ); + GEOS_ASSERT_EQ( numFaces, std::size( fg2l ) ); + GEOS_ASSERT_EQ( numFaces, std::size( fl2g ) ); + auto [l2g, g2l] = convertGlbLoc( fg2l ); array1d< integer > ghostRank_( numFaces ); @@ -242,12 +255,21 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, std::vector< integer > const & ghostRank, + std::map< NodeGlbIdx, std::array< double, 3 > > const & n2pos, std::map< NodeLocIdx, std::vector< EdgeLocIdx > > const & n2e, std::map< NodeLocIdx, std::vector< FaceLocIdx > > const & n2f, std::map< NodeLocIdx, std::vector< CellLocIdx > > const & n2c, std::map< NodeGlbIdx, NodeLocIdx > const & ng2l, std::map< NodeLocIdx, NodeGlbIdx > const & nl2g ) { + GEOS_ASSERT_EQ( numNodes, std::size( ghostRank ) ); + GEOS_ASSERT_EQ( numNodes, std::size( n2pos ) ); + GEOS_ASSERT_EQ( numNodes, std::size( n2e ) ); + GEOS_ASSERT_EQ( numNodes, std::size( n2f ) ); + GEOS_ASSERT_EQ( numNodes, std::size( n2c ) ); + GEOS_ASSERT_EQ( numNodes, std::size( ng2l ) ); + GEOS_ASSERT_EQ( numNodes, std::size( nl2g ) ); + // TODO MISSING cell type. Should be OK, the information is conveyed. // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock auto [l2g, g2l] = convertGlbLoc( ng2l ); @@ -258,7 +280,16 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, ghostRank_.emplace_back( gr ); } - return NodeMgrImpl( intConv< localIndex >( numNodes ), std::move( ghostRank_ ), convertToAoA( n2e ), convertToAoA( n2f ), convertToAoA( n2c ), std::move( l2g ) ); + array2d< real64, nodes::REFERENCE_POSITION_PERM > positions( numNodes, 3 ); + for( auto const & [ngi, pos]: n2pos ) + { + NodeLocIdx const nli = ng2l.at( ngi ); + positions( nli.get(), 0 ) = pos[0]; + positions( nli.get(), 1 ) = pos[2]; + positions( nli.get(), 2 ) = pos[2]; + } + + return NodeMgrImpl( intConv< localIndex >( numNodes ), std::move( positions ), std::move( ghostRank_ ), convertToAoA( n2e ), convertToAoA( n2f ), convertToAoA( n2c ), std::move( l2g ) ); } CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, @@ -269,6 +300,13 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, std::map< CellGlbIdx, CellLocIdx > const & cg2l, std::map< CellLocIdx, CellGlbIdx > const & cl2g ) { + GEOS_ASSERT_EQ( numCells, std::size( ghostRank ) ); + GEOS_ASSERT_EQ( numCells, std::size( c2n ) ); + GEOS_ASSERT_EQ( numCells, std::size( c2e ) ); + GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); + GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); + GEOS_ASSERT_EQ( numCells, std::size( cl2g ) ); + // TODO MISSING cell type. Should be OK, the information is conveyed. // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock auto [l2g, g2l] = convertGlbLoc( cg2l ); @@ -440,6 +478,7 @@ void buildPods( MeshGraph const & owned, NodeMgrImpl const nodeMgr = makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), buildGhostRank( g2l.nodes, send.nodes, recv.nodes ), + graph.n2pos, upwardMappings.n2e, upwardMappings.n2f, upwardMappings.n2c, diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 961eac8d05a..6b191bf0d08 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -18,12 +18,14 @@ namespace geos { NodeMgrImpl::NodeMgrImpl( localIndex numNodes, + array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, array1d< integer > && ghostRank, ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, array1d< globalIndex > const & l2g ) : m_numNodes( numNodes ), + m_positions( positions ), m_ghostRank( ghostRank ), m_n2e( n2e ), m_n2f( n2f ), @@ -38,7 +40,7 @@ localIndex NodeMgrImpl::numNodes() const array2d< real64, nodes::REFERENCE_POSITION_PERM > NodeMgrImpl::getNodePositions() const { - return {}; + return m_positions; } ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToEdges() const diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 47ddf4c571d..48afe36ed53 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -30,6 +30,7 @@ class NodeMgrImpl : public generators::NodeMgr { public: NodeMgrImpl( localIndex numNodes, + array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, array1d< integer > && ghostRank, ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, @@ -52,6 +53,7 @@ class NodeMgrImpl : public generators::NodeMgr private: localIndex m_numNodes; + array2d< real64, nodes::REFERENCE_POSITION_PERM > m_positions; array1d< integer > m_ghostRank; ArrayOfArrays< localIndex > m_n2e; ArrayOfArrays< localIndex > m_n2f; From 3ccdb41a6f7b829bba0b1df4caddfb504f79fbd6 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 6 Jun 2024 14:24:16 -0700 Subject: [PATCH 090/106] Start bringing the new mappings to the core of geos. --- .../mainInterface/ProblemManager.cpp | 46 +++++++++--- .../mainInterface/ProblemManager.hpp | 9 ++- src/coreComponents/mesh/MeshBody.hpp | 7 ++ src/coreComponents/mesh/MeshManager.cpp | 6 +- src/coreComponents/mesh/MeshManager.hpp | 2 +- .../mesh/generators/BuildPods.cpp | 20 +++-- .../mesh/generators/BuildPods.hpp | 6 +- .../mesh/generators/MeshGeneratorBase.cpp | 18 ++++- .../mesh/generators/MeshGeneratorBase.hpp | 12 ++- .../mesh/generators/NewGhosting.cpp | 16 ++-- .../mesh/generators/NewGhosting.hpp | 6 +- src/coreComponents/mesh/generators/Pods.cpp | 25 +++++++ src/coreComponents/mesh/generators/Pods.hpp | 73 +++++++++++++++++++ .../mesh/generators/VTKMeshGenerator.cpp | 40 +++++++--- .../mesh/generators/VTKMeshGenerator.hpp | 6 +- .../mesh/generators/include/CellMgr.hpp | 71 +++++++++--------- .../meshTests/testMeshGeneration.cpp | 2 +- .../unitTests/meshTests/testVTKImport.cpp | 2 +- 18 files changed, 274 insertions(+), 93 deletions(-) diff --git a/src/coreComponents/mainInterface/ProblemManager.cpp b/src/coreComponents/mainInterface/ProblemManager.cpp index 0d86412e41f..77e7fba2732 100644 --- a/src/coreComponents/mainInterface/ProblemManager.cpp +++ b/src/coreComponents/mainInterface/ProblemManager.cpp @@ -40,6 +40,7 @@ #include "mesh/DomainPartition.hpp" #include "mesh/MeshBody.hpp" #include "mesh/MeshManager.hpp" +#include "mesh/generators/include/MeshMappings.hpp" #include "mesh/simpleGeometricObjects/GeometricObjectManager.hpp" #include "mesh/mpiCommunications/CommunicationTools.hpp" #include "mesh/mpiCommunications/SpatialPartition.hpp" @@ -137,6 +138,11 @@ ProblemManager::ProblemManager( conduit::Node & root ): setRestartFlags( RestartFlags::WRITE ). setDescription( "Whether to disallow using pinned memory allocations for MPI communication buffers." ); + m_useNewGhosting = 1; +// registerWrapper( viewKeysStruct::useNewGhostingString(), &m_useNewGhosting ). +// setInputFlag( InputFlags::OPTIONAL ). +// setApplyDefaultValue( 0 ). +// setDescription( "Controls the use of the new ghosting implementation." ); } ProblemManager::~ProblemManager() @@ -566,6 +572,17 @@ void ProblemManager::initializationOrder( string_array & order ) } +void generateMeshLevelFreeFct( generators::MeshMappings const & meshMappings, + MeshLevel & meshLevel ) +{ + NodeManager & nodeManager = meshLevel.getNodeManager(); + EdgeManager & edgeManager = meshLevel.getEdgeManager(); + FaceManager & faceManager = meshLevel.getFaceManager(); + ElementRegionManager & elemRegionManager = meshLevel.getElemManager(); + + GEOS_ERROR( "Implementation in progess" ); +} + void ProblemManager::generateMesh() { GEOS_MARK_FUNCTION; @@ -573,7 +590,8 @@ void ProblemManager::generateMesh() MeshManager & meshManager = this->getGroup< MeshManager >( groupKeys.meshManager ); - meshManager.generateMeshes( domain ); + GEOS_LOG_RANK( "m_useNewGhosting = " << m_useNewGhosting ); + meshManager.generateMeshes( m_useNewGhosting, domain ); // get all the discretizations from the numerical methods. // map< pair< mesh body name, pointer to discretization>, array of region names > @@ -596,15 +614,24 @@ void ProblemManager::generateMesh() } else { - CellBlockManagerABC & cellBlockManager = meshBody.getGroup< CellBlockManagerABC >( keys::cellManager ); + if( m_useNewGhosting ) + { + GEOS_LOG_RANK_0( "Generating the mesh levels for the new ghosting." ); + generateMeshLevelFreeFct( meshBody.getMeshMappings(), baseMesh ); + // TODO add wells + } + else + { + CellBlockManagerABC & cellBlockManager = meshBody.getGroup< CellBlockManagerABC >( keys::cellManager ); - this->generateMeshLevel( baseMesh, - cellBlockManager, - nullptr, - junk.toViewConst() ); + this->generateMeshLevel( baseMesh, + cellBlockManager, + nullptr, + junk.toViewConst() ); - ElementRegionManager & elemManager = baseMesh.getElemManager(); - elemManager.generateWells( cellBlockManager, baseMesh ); + ElementRegionManager & elemManager = baseMesh.getElemManager(); + elemManager.generateWells( cellBlockManager, baseMesh ); + } } } ); @@ -630,14 +657,13 @@ void ProblemManager::generateMesh() int const order = feDiscretization->getOrder(); string const & discretizationName = feDiscretization->getName(); arrayView1d< string const > const regionNames = discretizationPair.second; - CellBlockManagerABC const & cellBlockManager = meshBody.getCellBlockManager(); // create a high order MeshLevel if( order > 1 ) { MeshLevel & mesh = meshBody.createMeshLevel( MeshBody::groupStructKeys::baseDiscretizationString(), discretizationName, order ); - + CellBlockManagerABC const & cellBlockManager = meshBody.getCellBlockManager(); this->generateMeshLevel( mesh, cellBlockManager, feDiscretization, diff --git a/src/coreComponents/mainInterface/ProblemManager.hpp b/src/coreComponents/mainInterface/ProblemManager.hpp index 00a453c34ea..24da2841f82 100644 --- a/src/coreComponents/mainInterface/ProblemManager.hpp +++ b/src/coreComponents/mainInterface/ProblemManager.hpp @@ -229,8 +229,10 @@ class ProblemManager : public dataRepository::Group dataRepository::ViewKey problemName = {"problemName"}; ///< Problem name key dataRepository::ViewKey outputDirectory = {"outputDirectory"}; ///< Output directory key dataRepository::ViewKey useNonblockingMPI = {"useNonblockingMPI"}; ///< Flag to use non-block MPI key - dataRepository::ViewKey suppressPinned = {"suppressPinned"}; ///< Flag to suppress use of pinned - ///< memory key + dataRepository::ViewKey suppressPinned = {"suppressPinned"}; ///< Flag to suppress use of pinned memory key + constexpr static char const * useNewGhostingString() + { return "useNewGhosting"; } + } viewKeys; ///< Command line input viewKeys /// Child group viewKeys @@ -383,6 +385,9 @@ class ProblemManager : public dataRepository::Group /// The FieldSpecificationManager FieldSpecificationManager * m_fieldSpecificationManager; + + /// Whether we should use the new ghosting implementation. + integer m_useNewGhosting = 0; }; } /* namespace geos */ diff --git a/src/coreComponents/mesh/MeshBody.hpp b/src/coreComponents/mesh/MeshBody.hpp index be1992183ce..f99670ec949 100644 --- a/src/coreComponents/mesh/MeshBody.hpp +++ b/src/coreComponents/mesh/MeshBody.hpp @@ -20,6 +20,8 @@ #define GEOS_MESH_MESHBODY_HPP_ #include "MeshLevel.hpp" + +#include "mesh/generators/include/MeshMappings.hpp" #include "dataRepository/KeyNames.hpp" namespace geos @@ -190,6 +192,11 @@ class MeshBody : public dataRepository::Group return this->getGroup< CellBlockManagerABC >( dataRepository::keys::cellManager ); } + generators::MeshMappings const & getMeshMappings() const + { + return this->getGroup< generators::MeshMappings >( "MeshMappingImpl" ); + } + /** * @brief De register the CellBlockManager from this meshBody */ diff --git a/src/coreComponents/mesh/MeshManager.cpp b/src/coreComponents/mesh/MeshManager.cpp index d1e48b4f419..1a1058147e8 100644 --- a/src/coreComponents/mesh/MeshManager.cpp +++ b/src/coreComponents/mesh/MeshManager.cpp @@ -58,7 +58,7 @@ void MeshManager::expandObjectCatalogs() } -void MeshManager::generateMeshes( DomainPartition & domain ) +void MeshManager::generateMeshes( bool useNewGhosting, DomainPartition & domain ) { forSubGroups< MeshGeneratorBase >( [&]( MeshGeneratorBase & meshGen ) { @@ -66,9 +66,9 @@ void MeshManager::generateMeshes( DomainPartition & domain ) meshBody.createMeshLevel( 0 ); SpatialPartition & partition = dynamic_cast< SpatialPartition & >(domain.getReference< PartitionBase >( keys::partitionManager ) ); - meshGen.generateMesh( meshBody, partition ); + meshGen.generateMesh( useNewGhosting, meshBody, partition ); - if( !meshBody.hasParticles() ) + if( !meshBody.hasParticles() && !useNewGhosting ) { CellBlockManagerABC const & cellBlockManager = meshBody.getCellBlockManager(); diff --git a/src/coreComponents/mesh/MeshManager.hpp b/src/coreComponents/mesh/MeshManager.hpp index b7aac9276bc..d9b97dcba78 100644 --- a/src/coreComponents/mesh/MeshManager.hpp +++ b/src/coreComponents/mesh/MeshManager.hpp @@ -60,7 +60,7 @@ class MeshManager : public dataRepository::Group * @brief Generate the meshes of the physical DomainPartition. * @param[in] domain a reference to the physical domain */ - void generateMeshes( DomainPartition & domain ); + void generateMeshes( bool useNewGhosting, DomainPartition & domain ); /** * @brief Generate the different mesh levels in a MeshBody of the domain. diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 4c720b937ff..999dba87e13 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -457,21 +457,12 @@ void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, GhostRecv const & recv, - GhostSend const & send ) + GhostSend const & send, + MeshMappingImpl & meshMappings ) { MeshGraph const graph = mergeMeshGraph( owned, present, ghosts ); auto const [g2l, l2g] = buildL2GMappings( graph ); -// GEOS_LOG_RANK( "numberings ng2l = " << json( numberings.ng2l ) ); -// GEOS_LOG_RANK( "owned mesh graph nodes = " << json( owned.n ) ); -// GEOS_LOG_RANK( "present mesh graph nodes = " << json( present.n ) ); -// GEOS_LOG_RANK( "ghosts mesh graph nodes = " << json( ghosts.n ) ); -// GEOS_LOG_RANK( "owned mesh graph e2n = " << json( owned.e2n ) ); -// GEOS_LOG_RANK( "present mesh graph e2n = " << json( present.e2n ) ); -// GEOS_LOG_RANK( "ghosts mesh graph e2n = " << json( ghosts.e2n ) ); -// auto const EdgeMgr = buildEdgeMgr( owned, present, ghosts, recv, send ); - -// MpiWrapper::barrier(); DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); @@ -507,6 +498,13 @@ void buildPods( MeshGraph const & owned, downwardMappings.c2f, g2l.cells, l2g.cells ); + + CellMgrImpl const cellMgr( cellBlock ); + + meshMappings.setCellMgr( cellMgr ); + meshMappings.setEdgeMgr( edgeMgr ); + meshMappings.setFaceMgr( faceMgr ); + meshMappings.setNodeMgr( nodeMgr ); } } diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index 60da63a5288..f522332958d 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -17,6 +17,9 @@ #include "Indices.hpp" +#include "include/MeshMappings.hpp" +#include "Pods.hpp" + #include #include #include @@ -86,7 +89,8 @@ void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, GhostRecv const & recv, - GhostSend const & send ); + GhostSend const & send, + MeshMappingImpl & meshMappings ); } diff --git a/src/coreComponents/mesh/generators/MeshGeneratorBase.cpp b/src/coreComponents/mesh/generators/MeshGeneratorBase.cpp index e085887ef3f..ca80c850bc2 100644 --- a/src/coreComponents/mesh/generators/MeshGeneratorBase.cpp +++ b/src/coreComponents/mesh/generators/MeshGeneratorBase.cpp @@ -15,6 +15,7 @@ #include "MeshGeneratorBase.hpp" #include "mesh/generators/CellBlockManager.hpp" #include "mesh/generators/ParticleBlockManager.hpp" +#include "mesh/generators/Pods.hpp" namespace geos { @@ -48,7 +49,7 @@ MeshGeneratorBase::CatalogInterface::CatalogType & MeshGeneratorBase::getCatalog return catalog; } -void MeshGeneratorBase::generateMesh( Group & parent, SpatialPartition & partition ) +void MeshGeneratorBase::generateMesh( bool useNewGhosting, Group & parent, SpatialPartition & partition ) { MeshBody & meshBody = dynamic_cast< MeshBody & >( parent ); if( meshBody.hasParticles() ) @@ -62,11 +63,20 @@ void MeshGeneratorBase::generateMesh( Group & parent, SpatialPartition & partiti } else { - CellBlockManager & cellBlockManager = parent.registerGroup< CellBlockManager >( keys::cellManager ); + if( useNewGhosting ) + { + MeshMappingImpl & mm = parent.registerGroup< MeshMappingImpl >( "MeshMappingImpl" ); - fillCellBlockManager( cellBlockManager, partition ); + fillMeshMappings( mm, partition ); + } + else + { + CellBlockManager & cellBlockManager = parent.registerGroup< CellBlockManager >( keys::cellManager ); - this->attachWellInfo( cellBlockManager ); + fillCellBlockManager( cellBlockManager, partition ); + + this->attachWellInfo( cellBlockManager ); + } } } diff --git a/src/coreComponents/mesh/generators/MeshGeneratorBase.hpp b/src/coreComponents/mesh/generators/MeshGeneratorBase.hpp index 48b97d15280..b93063f2334 100644 --- a/src/coreComponents/mesh/generators/MeshGeneratorBase.hpp +++ b/src/coreComponents/mesh/generators/MeshGeneratorBase.hpp @@ -21,6 +21,8 @@ #include "mesh/mpiCommunications/SpatialPartition.hpp" +#include "mesh/generators/Pods.hpp" + #include "dataRepository/Group.hpp" #include "dataRepository/WrapperBase.hpp" #include "codingUtilities/Utilities.hpp" @@ -78,7 +80,7 @@ class MeshGeneratorBase : public dataRepository::Group * @param parent The parent group of the CellBlockManager. * @param[in] partition The reference to spatial partition */ - void generateMesh( Group & parent, SpatialPartition & partition ); + void generateMesh( bool useNewGhosting, Group & parent, SpatialPartition & partition ); /** * @brief Describe which kind of block must be considered. @@ -144,6 +146,14 @@ class MeshGeneratorBase : public dataRepository::Group GEOS_ERROR( "Cell mesh generation not implemented for generator of this type" ); } + virtual void fillMeshMappings( MeshMappingImpl & meshMappings, + SpatialPartition & partition ) + { + GEOS_UNUSED_VAR( meshMappings ); + GEOS_UNUSED_VAR( partition ); + GEOS_ERROR( "Not Implemented, you should not be there!" ); + } + void attachWellInfo( CellBlockManager & cellBlockManager ); /** diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 437dfdd2fc0..8578d395b09 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1110,8 +1110,9 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & return { std::move( ghosts ), std::move( recv ), std::move( send ) }; } -std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< MpiRank > const & neighbors ) +void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< MpiRank > const & neighbors, + MeshMappingImpl & meshMappings ) { auto const [buckets, offsets] = doTheNewGlobalNumbering( mesh, neighbors ); @@ -1134,13 +1135,12 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v auto const [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); - buildPods( owned, present, ghosts, recv, send ); - - return {}; + buildPods( owned, present, ghosts, recv, send, meshMappings ); } -std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< int > const & neighbors ) +void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors, + MeshMappingImpl & meshMappings ) { std::set< MpiRank > neighbors_; for( int const & rank: neighbors ) @@ -1149,7 +1149,7 @@ std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< v } GEOS_LOG_RANK( "my initial neighbors are " << json( neighbors_ ) ); - return doTheNewGhosting( mesh, neighbors_ ); + return doTheNewGhosting( mesh, neighbors_, meshMappings ); } } // end of namespace geos::ghosting diff --git a/src/coreComponents/mesh/generators/NewGhosting.hpp b/src/coreComponents/mesh/generators/NewGhosting.hpp index b077e514126..11bfa7a5fac 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.hpp +++ b/src/coreComponents/mesh/generators/NewGhosting.hpp @@ -16,6 +16,7 @@ #define GEOS_NEWGHOSTING_HPP #include "include/MeshMappings.hpp" +#include "Pods.hpp" #include #include @@ -25,8 +26,9 @@ namespace geos::ghosting { -std::unique_ptr< generators::MeshMappings > doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, - std::set< int > const & neighbors ); +void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, + std::set< int > const & neighbors, + MeshMappingImpl & meshMappings ); } // end of namespace ghosting diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 6b191bf0d08..713afc5809b 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -239,4 +239,29 @@ array1d< globalIndex > CellBlkImpl::localToGlobalMap() const // return {}; //} +generators::CellMgr const & MeshMappingImpl::getCellMgr() const +{ + return m_cellMgr; +} + +generators::EdgeMgr const & MeshMappingImpl::getEdgeMgr() const +{ + return m_edgeMgr; +} + +generators::FaceMgr const & MeshMappingImpl::getFaceMgr() const +{ + return m_faceMgr; +} + +generators::NodeMgr const & MeshMappingImpl::getNodeMgr() const +{ + return m_nodeMgr; +} + +std::list< CellBlk const * > CellMgrImpl::getCellBlks() const +{ + return { &m_cellBlk }; +} + } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 48afe36ed53..1747c94103e 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -22,6 +22,7 @@ #include "include/FaceMgr.hpp" #include "include/CellMgr.hpp" #include "include/CellBlk.hpp" +#include "include/MeshMappings.hpp" namespace geos { @@ -29,6 +30,9 @@ namespace geos class NodeMgrImpl : public generators::NodeMgr { public: + NodeMgrImpl() + { } + NodeMgrImpl( localIndex numNodes, array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, array1d< integer > && ghostRank, @@ -65,6 +69,9 @@ class NodeMgrImpl : public generators::NodeMgr class EdgeMgrImpl : public generators::EdgeMgr { public: + EdgeMgrImpl() + { } + EdgeMgrImpl( std::size_t numEdges, array1d< integer > && ghostRank, array2d< localIndex > && e2n, @@ -94,6 +101,9 @@ class EdgeMgrImpl : public generators::EdgeMgr class FaceMgrImpl : public generators::FaceMgr { public: + FaceMgrImpl() + { } + FaceMgrImpl( std::size_t numFaces, array1d< integer > && ghostRank, ArrayOfArrays< localIndex > && f2n, @@ -127,6 +137,9 @@ class FaceMgrImpl : public generators::FaceMgr class CellBlkImpl : public CellBlk { public: + CellBlkImpl() + { } + CellBlkImpl( localIndex numCells, array1d< integer > const & ghostRank, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, @@ -161,6 +174,66 @@ class CellBlkImpl : public CellBlk array1d< globalIndex > m_l2g; }; +class CellMgrImpl : public generators::CellMgr +{ +public: + CellMgrImpl() + { } + + CellMgrImpl( CellBlkImpl const & cellBlks ) + : + m_cellBlk( cellBlks ) + { } + + [[nodiscard]] std::list< CellBlk const * > getCellBlks() const override; + +private: + CellBlkImpl m_cellBlk; +}; + +class MeshMappingImpl : public generators::MeshMappings +{ +public: + MeshMappingImpl( string const & name, + Group * const parent ) + : MeshMappings( name, parent ) + { } + + [[nodiscard]] generators::CellMgr const & getCellMgr() const override; + + [[nodiscard]] generators::EdgeMgr const & getEdgeMgr() const override; + + [[nodiscard]] generators::FaceMgr const & getFaceMgr() const override; + + [[nodiscard]] generators::NodeMgr const & getNodeMgr() const override; + + void setCellMgr( CellMgrImpl const & cellMgr ) + { + m_cellMgr = cellMgr; + } + + void setEdgeMgr( EdgeMgrImpl const & edgeMgr ) + { + m_edgeMgr = edgeMgr; + } + + void setFaceMgr( FaceMgrImpl const & faceMgr ) + { + m_faceMgr = faceMgr; + } + + void setNodeMgr( NodeMgrImpl const & nodeMgr ) + { + m_nodeMgr = nodeMgr; + } + +private: + CellMgrImpl m_cellMgr; + EdgeMgrImpl m_edgeMgr; + FaceMgrImpl m_faceMgr; + NodeMgrImpl m_nodeMgr; +}; + } // geos #endif //GEOS_PODS_HPP diff --git a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp index 3fe5fdd34dc..bda2033f301 100644 --- a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp +++ b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp @@ -77,11 +77,37 @@ VTKMeshGenerator::VTKMeshGenerator( string const & name, " If set to 0 (default value), the GlobalId arrays in the input mesh are used if available, and generated otherwise." " If set to a negative value, the GlobalId arrays in the input mesh are not used, and generated global Ids are automatically generated." " If set to a positive value, the GlobalId arrays in the input mesh are used and required, and the simulation aborts if they are not available" ); +} - registerWrapper( viewKeyStruct::useNewGhostingString(), &m_useNewGhosting ). - setInputFlag( InputFlags::OPTIONAL ). - setApplyDefaultValue( 0 ). - setDescription( "Controls the use of the new ghosting implementation." ); +void VTKMeshGenerator::fillMeshMappings( MeshMappingImpl & meshMappings, + SpatialPartition & partition ) +{ + // TODO duplicated from `fillCellBlockManager`. + MPI_Comm const comm = MPI_COMM_GEOSX; + vtkSmartPointer< vtkMultiProcessController > controller = vtk::getController(); + vtkMultiProcessController::SetGlobalController( controller ); + + GEOS_LOG_RANK_0( GEOS_FMT( "{} '{}': reading mesh from {}", catalogName(), getName(), m_filePath ) ); + { + + GEOS_LOG_LEVEL_RANK_0( 2, " reading the dataset..." ); + vtk::AllMeshes allMeshes = vtk::loadAllMeshes( m_filePath, m_mainBlockName, m_faceBlockNames ); + GEOS_LOG_LEVEL_RANK_0( 2, " redistributing mesh..." ); + vtk::AllMeshes redistributedMeshes = + vtk::redistributeMeshes( getLogLevel(), allMeshes.getMainMesh(), allMeshes.getFaceBlocks(), comm, m_partitionMethod, m_partitionRefinement, m_useGlobalIds ); + m_vtkMesh = redistributedMeshes.getMainMesh(); + m_faceBlockMeshes = redistributedMeshes.getFaceBlocks(); + GEOS_LOG_LEVEL_RANK_0( 2, " finding neighbor ranks..." ); + std::vector< vtkBoundingBox > boxes = vtk::exchangeBoundingBoxes( *m_vtkMesh, comm ); + std::vector< int > const neighbors = vtk::findNeighborRanks( std::move( boxes ) ); + partition.setMetisNeighborList( std::move( neighbors ) ); + GEOS_LOG_LEVEL_RANK_0( 2, " done!" ); + } + GEOS_LOG_RANK_0( GEOS_FMT( "{} '{}': generating GEOSX mesh data structure", catalogName(), getName() ) ); + + + GEOS_LOG_RANK( "Here we go, new ghosting!" ); + ghosting::doTheNewGhosting( m_vtkMesh, partition.getMetisNeighborList(), meshMappings ); } void VTKMeshGenerator::fillCellBlockManager( CellBlockManager & cellBlockManager, SpatialPartition & partition ) @@ -110,12 +136,6 @@ void VTKMeshGenerator::fillCellBlockManager( CellBlockManager & cellBlockManager } GEOS_LOG_RANK_0( GEOS_FMT( "{} '{}': generating GEOSX mesh data structure", catalogName(), getName() ) ); - if( m_useNewGhosting ) - { - GEOS_LOG_RANK( "Here we go, new ghosting!" ); - std::unique_ptr< generators::MeshMappings > mm = ghosting::doTheNewGhosting( m_vtkMesh, partition.getMetisNeighborList() ); - } - GEOS_LOG_LEVEL_RANK_0( 2, " preprocessing..." ); m_cellMap = vtk::buildCellMap( *m_vtkMesh, m_attributeName ); diff --git a/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp b/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp index 83845f27c42..723ecf8d32b 100644 --- a/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp +++ b/src/coreComponents/mesh/generators/VTKMeshGenerator.hpp @@ -98,6 +98,8 @@ class VTKMeshGenerator : public ExternalMeshGeneratorBase virtual void freeResources() override; private: + void fillMeshMappings( MeshMappingImpl & meshMappings, + SpatialPartition & partition ) override; ///@cond DO_NOT_DOCUMENT struct viewKeyStruct @@ -109,7 +111,6 @@ class VTKMeshGenerator : public ExternalMeshGeneratorBase constexpr static char const * partitionRefinementString() { return "partitionRefinement"; } constexpr static char const * partitionMethodString() { return "partitionMethod"; } constexpr static char const * useGlobalIdsString() { return "useGlobalIds"; } - constexpr static char const * useNewGhostingString() { return "useNewGhosting"; } }; /// @endcond @@ -150,9 +151,6 @@ class VTKMeshGenerator : public ExternalMeshGeneratorBase /// Whether global id arrays should be used, if available integer m_useGlobalIds = 0; - // Whether we should use the new ghosting implementation. - integer m_useNewGhosting = 0; - /// Method (library) used to partition the mesh vtk::PartitionMethod m_partitionMethod = vtk::PartitionMethod::parmetis; diff --git a/src/coreComponents/mesh/generators/include/CellMgr.hpp b/src/coreComponents/mesh/generators/include/CellMgr.hpp index 38e188e4fad..69013a6d822 100644 --- a/src/coreComponents/mesh/generators/include/CellMgr.hpp +++ b/src/coreComponents/mesh/generators/include/CellMgr.hpp @@ -16,6 +16,7 @@ #define GEOS_CELLMGR_HPP #include "LineBlockABC.hpp" +#include "CellBlk.hpp" #include "dataRepository/Group.hpp" @@ -28,40 +29,42 @@ namespace geos::generators class CellMgr { public: - /** - * @brief Returns a group containing the cell blocks as @p CellBlockABC instances. - * @return Mutable reference to the cell blocks group. - * - * @note It should probably be better not to expose a non-const accessor here. - */ - virtual dataRepository::Group & getCellBlocks() = 0; - - /** - * @brief Returns a group containing the face blocks as @p FaceBlockABC instances. - * @return Mutable reference to the face blocks group. - * - * @note It should probably be better not to expose a non-const accessor here. - */ - virtual dataRepository::Group & getFaceBlocks() = 0; - - /** - * @brief Returns LineBlockABC corresponding to the given identifier - * @param name the name of the required LineBlockABC - * @return The LineBlockABC associated with the given name - */ - virtual LineBlockABC const & getLineBlock( string name ) const = 0; - - /** - * @brief Returns a group containing the cell blocks as CellBlockABC instances - * @return Const reference to the Group instance. - */ - virtual const dataRepository::Group & getCellBlocks() const = 0; - - /** - * @brief Returns a group containing the face blocks as FaceBlockABC instances - * @return Const reference to the Group instance. - */ - virtual const dataRepository::Group & getFaceBlocks() const = 0; + virtual std::list< CellBlk const * > getCellBlks() const = 0; +// +// /** +// * @brief Returns a group containing the cell blocks as @p CellBlockABC instances. +// * @return Mutable reference to the cell blocks group. +// * +// * @note It should probably be better not to expose a non-const accessor here. +// */ +// virtual dataRepository::Group & getCellBlocks() = 0; +// +// /** +// * @brief Returns a group containing the face blocks as @p FaceBlockABC instances. +// * @return Mutable reference to the face blocks group. +// * +// * @note It should probably be better not to expose a non-const accessor here. +// */ +// virtual dataRepository::Group & getFaceBlocks() = 0; +// +// /** +// * @brief Returns LineBlockABC corresponding to the given identifier +// * @param name the name of the required LineBlockABC +// * @return The LineBlockABC associated with the given name +// */ +// virtual LineBlockABC const & getLineBlock( string name ) const = 0; +// +// /** +// * @brief Returns a group containing the cell blocks as CellBlockABC instances +// * @return Const reference to the Group instance. +// */ +// virtual const dataRepository::Group & getCellBlocks() const = 0; +// +// /** +// * @brief Returns a group containing the face blocks as FaceBlockABC instances +// * @return Const reference to the Group instance. +// */ +// virtual const dataRepository::Group & getFaceBlocks() const = 0; }; } diff --git a/src/coreComponents/unitTests/meshTests/testMeshGeneration.cpp b/src/coreComponents/unitTests/meshTests/testMeshGeneration.cpp index e2891e9eb96..ac196c5f476 100644 --- a/src/coreComponents/unitTests/meshTests/testMeshGeneration.cpp +++ b/src/coreComponents/unitTests/meshTests/testMeshGeneration.cpp @@ -523,7 +523,7 @@ TEST_F( MeshGenerationTest, highOrderMapsSizes ) DomainPartition & domain = problemManager.getDomainPartition(); MeshBody & meshBody = domain.getMeshBody( 0 ); MeshManager & meshManager = problemManager.getGroup< MeshManager >( problemManager.groupKeys.meshManager ); - meshManager.generateMeshes( domain ); + meshManager.generateMeshes( false, domain ); for( int order = minOrder; order < maxOrder; order++ ) { MeshLevel & meshLevel = meshBody.createMeshLevel( MeshBody::groupStructKeys::baseDiscretizationString(), GEOS_FMT( "TestLevel{}", order ), order ); diff --git a/src/coreComponents/unitTests/meshTests/testVTKImport.cpp b/src/coreComponents/unitTests/meshTests/testVTKImport.cpp index cd3a9c00384..8fda0d13d0b 100644 --- a/src/coreComponents/unitTests/meshTests/testVTKImport.cpp +++ b/src/coreComponents/unitTests/meshTests/testVTKImport.cpp @@ -71,7 +71,7 @@ void TestMeshImport( string const & meshFilePath, V const & validate, string con meshManager.processInputFileRecursive( xmlDocument, xmlMeshNode ); meshManager.postProcessInputRecursive(); DomainPartition domain( "domain", &root ); - meshManager.generateMeshes( domain ); + meshManager.generateMeshes( false, domain ); // TODO Field import is not tested yet. Proper refactoring needs to be done first. From 2382685b4c044c759f0707c6021605a250dca7dc Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:51:19 -0700 Subject: [PATCH 091/106] First end-to-end version that runs Laplace in parallel. --- .../mainInterface/ProblemManager.cpp | 65 +++- src/coreComponents/mesh/CMakeLists.txt | 1 + src/coreComponents/mesh/CellElementRegion.cpp | 13 + src/coreComponents/mesh/CellElementRegion.hpp | 9 + .../mesh/CellElementSubRegion.cpp | 36 +++ .../mesh/CellElementSubRegion.hpp | 8 + src/coreComponents/mesh/DomainPartition.cpp | 5 +- src/coreComponents/mesh/EdgeManager.cpp | 17 ++ src/coreComponents/mesh/EdgeManager.hpp | 8 + .../mesh/ElementRegionManager.cpp | 37 +++ .../mesh/ElementRegionManager.hpp | 5 + src/coreComponents/mesh/FaceManager.cpp | 28 ++ src/coreComponents/mesh/FaceManager.hpp | 13 + src/coreComponents/mesh/NodeManager.cpp | 30 ++ src/coreComponents/mesh/NodeManager.hpp | 9 + src/coreComponents/mesh/ObjectManagerBase.cpp | 20 ++ src/coreComponents/mesh/ObjectManagerBase.hpp | 11 + .../mesh/generators/BuildPods.cpp | 279 +++++++++++++---- .../mesh/generators/BuildPods.hpp | 2 +- .../mesh/generators/NewGhosting.cpp | 37 ++- src/coreComponents/mesh/generators/Pods.cpp | 184 ++---------- src/coreComponents/mesh/generators/Pods.hpp | 283 ++++++++++++++---- .../mesh/generators/VTKMeshGenerator.cpp | 1 - .../mesh/generators/include/CellBlk.hpp | 12 +- .../mesh/generators/include/CellMgr.hpp | 2 +- .../mesh/generators/include/EdgeMgr.hpp | 13 +- .../mesh/generators/include/FaceMgr.hpp | 8 +- .../mesh/generators/include/GhostExchange.hpp | 59 ++++ .../mesh/generators/include/MeshMappings.hpp | 2 + .../mesh/generators/include/NodeMgr.hpp | 10 +- 30 files changed, 873 insertions(+), 334 deletions(-) create mode 100644 src/coreComponents/mesh/generators/include/GhostExchange.hpp diff --git a/src/coreComponents/mainInterface/ProblemManager.cpp b/src/coreComponents/mainInterface/ProblemManager.cpp index 77e7fba2732..8407b05590f 100644 --- a/src/coreComponents/mainInterface/ProblemManager.cpp +++ b/src/coreComponents/mainInterface/ProblemManager.cpp @@ -48,6 +48,10 @@ #include "physicsSolvers/SolverBase.hpp" #include "schema/schemaUtilities.hpp" +#include + +using json = nlohmann::json; + // System includes #include #include @@ -168,7 +172,8 @@ void ProblemManager::problemSetup() initialize(); - importFields(); + // TODO Import the fields back! +// importFields(); } @@ -573,6 +578,7 @@ void ProblemManager::initializationOrder( string_array & order ) void generateMeshLevelFreeFct( generators::MeshMappings const & meshMappings, + GeometricObjectManager const & geometries, MeshLevel & meshLevel ) { NodeManager & nodeManager = meshLevel.getNodeManager(); @@ -580,7 +586,46 @@ void generateMeshLevelFreeFct( generators::MeshMappings const & meshMappings, FaceManager & faceManager = meshLevel.getFaceManager(); ElementRegionManager & elemRegionManager = meshLevel.getElemManager(); - GEOS_ERROR( "Implementation in progess" ); + elemRegionManager.generateMesh( meshMappings ); + nodeManager.setGeometricalRelations( meshMappings, elemRegionManager ); + edgeManager.setGeometricalRelations( meshMappings.getNeighbors(), meshMappings.getEdgeMgr() ); + faceManager.setGeometricalRelations( meshMappings, elemRegionManager, nodeManager ); +// nodeManager.constructGlobalToLocalMap( cellBlockManager ); +// TODO Still need to work on the sets. +// // Edge, face and element region managers rely on the sets provided by the node manager. +// // This is why `nodeManager.buildSets` is called first. +// nodeManager.buildSets( cellBlockManager, this->getGroup< GeometricObjectManager >( groupKeys.geometricObjectManager ) ); + nodeManager.buildGeometricSets( geometries ); + edgeManager.buildSets( nodeManager ); + faceManager.buildSets( nodeManager ); + elemRegionManager.buildSets( nodeManager ); + // The edge manager do not hold any information related to the regions nor the elements. + // This is why the element region manager is not provided. + nodeManager.setupRelatedObjectsInRelations( edgeManager, faceManager, elemRegionManager ); + edgeManager.setupRelatedObjectsInRelations( nodeManager, faceManager ); + faceManager.setupRelatedObjectsInRelations( nodeManager, edgeManager, elemRegionManager ); + // FIXME Boundary markers should now be obsolete +// // Node and edge managers rely on the boundary information provided by the face manager. +// // This is why `faceManager.setDomainBoundaryObjects` is called first. +// faceManager.setDomainBoundaryObjects( elemRegionManager ); +// edgeManager.setDomainBoundaryObjects( faceManager ); +// nodeManager.setDomainBoundaryObjects( faceManager, edgeManager ); +// + meshLevel.generateSets(); +// + elemRegionManager.forElementSubRegions< ElementSubRegionBase >( [&]( ElementSubRegionBase & subRegion ) + { + subRegion.setupRelatedObjectsInRelations( meshLevel ); + // `FaceElementSubRegion` has no node and therefore needs the nodes positions from the neighbor elements + // in order to compute the geometric quantities. + // And this point of the process, the ghosting has not been done and some elements of the `FaceElementSubRegion` + // can have no neighbor. Making impossible the computation, which is therefore postponed to after the ghosting. + subRegion.calculateElementGeometricQuantities( nodeManager, faceManager ); +// subRegion.setMaxGlobalIndex(); // FIXME This should be useless for static meshes (maybe needed for surface generator). + } ); +// elemRegionManager.setMaxGlobalIndex(); + +// GEOS_ERROR( "Implementation in progress" ); } void ProblemManager::generateMesh() @@ -595,8 +640,7 @@ void ProblemManager::generateMesh() // get all the discretizations from the numerical methods. // map< pair< mesh body name, pointer to discretization>, array of region names > - map< std::pair< string, Group const * const >, arrayView1d< string const > const > - discretizations = getDiscretizations(); + map< std::pair< string, Group const * const >, arrayView1d< string const > const > discretizations = getDiscretizations(); // setup the base discretizations (hard code this for now) domain.forMeshBodies( [&]( MeshBody & meshBody ) @@ -617,8 +661,14 @@ void ProblemManager::generateMesh() if( m_useNewGhosting ) { GEOS_LOG_RANK_0( "Generating the mesh levels for the new ghosting." ); - generateMeshLevelFreeFct( meshBody.getMeshMappings(), baseMesh ); + generateMeshLevelFreeFct( meshBody.getMeshMappings(), + this->getGroup< GeometricObjectManager >( groupKeys.geometricObjectManager ), + baseMesh ); // TODO add wells + for( integer const & neighbor: meshBody.getMeshMappings().getNeighbors() ) + { + domain.getNeighbors().emplace_back( neighbor ); + } } else { @@ -637,7 +687,7 @@ void ProblemManager::generateMesh() Group const & commandLine = this->getGroup< Group >( groupKeys.commandLine ); integer const useNonblockingMPI = commandLine.getReference< integer >( viewKeys.useNonblockingMPI ); - domain.setupBaseLevelMeshGlobalInfo(); +// domain.setupBaseLevelMeshGlobalInfo(); // setup the MeshLevel associated with the discretizations for( auto const & discretizationPair: discretizations ) @@ -649,7 +699,7 @@ void ProblemManager::generateMesh() { // particle mesh bodies don't have a finite element // discretization FiniteElementDiscretization const * const - feDiscretization = dynamic_cast< FiniteElementDiscretization const * >( discretizationPair.first.second ); + feDiscretization = dynamic_cast< FiniteElementDiscretization const * >( discretizationPair.first.second ); // if the discretization is a finite element discretization if( feDiscretization != nullptr ) @@ -730,7 +780,6 @@ void ProblemManager::generateMesh() edgeManager.setIsExternal( faceManager ); } ); } ); - } diff --git a/src/coreComponents/mesh/CMakeLists.txt b/src/coreComponents/mesh/CMakeLists.txt index c62b41f2974..ca4b6d9cdf3 100644 --- a/src/coreComponents/mesh/CMakeLists.txt +++ b/src/coreComponents/mesh/CMakeLists.txt @@ -63,6 +63,7 @@ set( mesh_headers generators/include/CellBlk.hpp generators/include/CellBlockABC.hpp generators/include/FaceBlockABC.hpp + generators/include/GhostExchange.hpp generators/include/LineBlockABC.hpp mpiCommunications/CommID.hpp mpiCommunications/CommunicationTools.hpp diff --git a/src/coreComponents/mesh/CellElementRegion.cpp b/src/coreComponents/mesh/CellElementRegion.cpp index f10494109b0..dca59309d23 100644 --- a/src/coreComponents/mesh/CellElementRegion.cpp +++ b/src/coreComponents/mesh/CellElementRegion.cpp @@ -47,6 +47,19 @@ void CellElementRegion::generateMesh( Group const & cellBlocks ) } } +void CellElementRegion::generateMesh( std::set< integer > const & neighbors, + std::map< string, generators::CellBlk const * > const & cellBlks ) +{ + Group & elementSubRegions = this->getGroup( viewKeyStruct::elementSubRegions() ); + + for( string const & cellBlockName: this->m_cellBlockNames ) + { + CellElementSubRegion & subRegion = elementSubRegions.registerGroup< CellElementSubRegion >( cellBlockName ); + generators::CellBlk const * source = cellBlks.at( cellBlockName ); + subRegion.copyFromCellBlock( neighbors, *source ); + } +} + REGISTER_CATALOG_ENTRY( ObjectManagerBase, CellElementRegion, string const &, Group * const ) } /* namespace geos */ diff --git a/src/coreComponents/mesh/CellElementRegion.hpp b/src/coreComponents/mesh/CellElementRegion.hpp index 130423b42bc..2aec4f23979 100644 --- a/src/coreComponents/mesh/CellElementRegion.hpp +++ b/src/coreComponents/mesh/CellElementRegion.hpp @@ -21,6 +21,8 @@ #include "ElementRegionBase.hpp" +#include "mesh/generators/include/CellBlk.hpp" + namespace geos { @@ -115,6 +117,13 @@ class CellElementRegion : public ElementRegionBase virtual void generateMesh( Group const & cellBlocks ) override; + /** + * @brief Fill the current @c CellElementRegion with the cell blocks. + * @param neighbors The neighbors of the current @c CellElementRegion + * @param cellBlks All the cell blocks. It's the role of the current @c CellElementRegion to pick the one it needs. + */ + void generateMesh( std::set< integer > const & neighbors, std::map< string, generators::CellBlk const * > const & cellBlks ); + ///@} /** diff --git a/src/coreComponents/mesh/CellElementSubRegion.cpp b/src/coreComponents/mesh/CellElementSubRegion.cpp index 550b3614871..1cb6fb931e9 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.cpp +++ b/src/coreComponents/mesh/CellElementSubRegion.cpp @@ -19,6 +19,11 @@ #include "mesh/MeshLevel.hpp" #include "mesh/generators/CellBlockUtilities.hpp" +#include "common/MpiWrapper.hpp" + +#include +using json = nlohmann::json; + namespace geos { using namespace dataRepository; @@ -101,6 +106,37 @@ void CellElementSubRegion::copyFromCellBlock( CellBlockABC const & cellBlock ) } ); } +void CellElementSubRegion::copyFromCellBlock( std::set< integer > const & neighbors, + generators::CellBlk const & cellBlock ) +{ + // Defines the (unique) element type of this cell element region, + // and its associated number of nodes, edges, faces. + m_elementType = cellBlock.getElementType(); + m_numNodesPerElement = cellBlock.numNodesPerElement(); + m_numEdgesPerElement = cellBlock.numEdgesPerElement(); + m_numFacesPerElement = cellBlock.numFacesPerElement(); + + // We call the `resize` member function of the cell to (nodes, edges, faces) relations, + // before calling the `CellElementSubRegion::resize` in order to keep the first dimension. + // Be careful when refactoring. + m_toNodesRelation.resize( this->size(), m_numNodesPerElement ); + m_toEdgesRelation.resize( this->size(), m_numEdgesPerElement ); + m_toFacesRelation.resize( this->size(), m_numFacesPerElement ); + this->resize( cellBlock.numElements() ); + + this->nodeList() = cellBlock.getElemToNodes(); + this->edgeList() = cellBlock.getElemToEdges(); + this->faceList() = cellBlock.getElemToFaces(); + + this->m_localToGlobalMap = cellBlock.getLocalToGlobal(); + this->m_globalToLocalMap = cellBlock.getGlobalToLocal(); + + copyExchangeInfo( neighbors, cellBlock.getGhostRank(), cellBlock.getSend(), cellBlock.getRecv() ); + + //this->constructGlobalToLocalMap(); // TODO what about `m_localMaxGlobalIndex`? + // TODO add the external properties import +} + void CellElementSubRegion::addFracturedElement( localIndex const cellElemIndex, localIndex const embSurfIndex ) { diff --git a/src/coreComponents/mesh/CellElementSubRegion.hpp b/src/coreComponents/mesh/CellElementSubRegion.hpp index 758a8fa0913..69914cc61af 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.hpp +++ b/src/coreComponents/mesh/CellElementSubRegion.hpp @@ -17,6 +17,7 @@ #define GEOS_MESH_CELLELEMENTSUBREGION_HPP_ #include "mesh/generators/include/CellBlockABC.hpp" +#include "mesh/generators/include/CellBlk.hpp" #include "mesh/NodeManager.hpp" #include "mesh/FaceManager.hpp" #include "mesh/utilities/ComputationalGeometry.hpp" @@ -84,6 +85,13 @@ class CellElementSubRegion : public ElementSubRegionBase */ void copyFromCellBlock( CellBlockABC const & cellBlock ); + /** + * @brief Fill the current @c CellElementSubRegion using the @c cellBlock information + * @param[in] neighbors The neighbors' ranks. + * @param[in] cellBlock The cell block with all the mappings and ghosting information. + */ + void copyFromCellBlock( std::set< integer > const & neighbors, generators::CellBlk const & cellBlock ); + ///@} /** diff --git a/src/coreComponents/mesh/DomainPartition.cpp b/src/coreComponents/mesh/DomainPartition.cpp index 50cecbc24d9..fb95a1e4385 100644 --- a/src/coreComponents/mesh/DomainPartition.cpp +++ b/src/coreComponents/mesh/DomainPartition.cpp @@ -243,8 +243,9 @@ void DomainPartition::setupCommunications( bool use_nonblocking ) NodeManager & nodeManager = meshLevel.getNodeManager(); FaceManager & faceManager = meshLevel.getFaceManager(); - CommunicationTools::getInstance().setupGhosts( meshLevel, m_neighbors, use_nonblocking ); - faceManager.sortAllFaceNodes( nodeManager, meshLevel.getElemManager() ); + // TODO Make this optional instead of commented out! +// CommunicationTools::getInstance().setupGhosts( meshLevel, m_neighbors, use_nonblocking ); +// faceManager.sortAllFaceNodes( nodeManager, meshLevel.getElemManager() ); faceManager.computeGeometry( nodeManager ); } else if( !meshLevel.isShallowCopyOf( meshBody.getMeshLevels().getGroup< MeshLevel >( 0 )) ) diff --git a/src/coreComponents/mesh/EdgeManager.cpp b/src/coreComponents/mesh/EdgeManager.cpp index 108676e1f25..4140104371e 100644 --- a/src/coreComponents/mesh/EdgeManager.cpp +++ b/src/coreComponents/mesh/EdgeManager.cpp @@ -105,6 +105,23 @@ void EdgeManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); } +void EdgeManager::setGeometricalRelations( std::set< integer > const & neighbors, + generators::EdgeMgr const & edgeMgr ) +{ + GEOS_MARK_FUNCTION; + resize( edgeMgr.numEdges() ); + + m_toNodesRelation.base() = edgeMgr.getEdgeToNodes(); + m_toFacesRelation.base().assimilate< parallelHostPolicy >( edgeMgr.getEdgeToFaces(), + LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); + // TODO This is new + m_localToGlobalMap = edgeMgr.getLocalToGlobal(); + m_globalToLocalMap = edgeMgr.getGlobalToLocal(); + + // TODO not for there, but it's convenient + copyExchangeInfo( neighbors, edgeMgr.getGhostRank(), edgeMgr.getSend(), edgeMgr.getRecv() ); +} + void EdgeManager::setupRelatedObjectsInRelations( NodeManager const & nodeManager, FaceManager const & faceManager ) { diff --git a/src/coreComponents/mesh/EdgeManager.hpp b/src/coreComponents/mesh/EdgeManager.hpp index 0eedb9a0870..c332f51b544 100644 --- a/src/coreComponents/mesh/EdgeManager.hpp +++ b/src/coreComponents/mesh/EdgeManager.hpp @@ -21,6 +21,7 @@ #include "mesh/ObjectManagerBase.hpp" #include "mesh/generators/CellBlockManagerABC.hpp" +#include "mesh/generators/include/EdgeMgr.hpp" #include "InterObjectRelation.hpp" #include "ToElementRelation.hpp" #include "LvArray/src/tensorOps.hpp" @@ -127,6 +128,13 @@ class EdgeManager : public ObjectManagerBase */ void setGeometricalRelations( CellBlockManagerABC const & cellBlockManager, bool isBaseMeshLevel ); + /** + * @brief Initialise the current @c EdgeManager with the information from @c edgeMgr. + * @param[in] neighbors The ranks of the neighbors. + * @param[in] edgeMgr Provides the geometrical mappings at stake for the @c EdgeManager. + */ + void setGeometricalRelations( std::set< integer > const & neighbors, generators::EdgeMgr const & edgeMgr ); + /** * @brief Link the current manager to other managers. * @param[in] nodeManager The node manager instance. diff --git a/src/coreComponents/mesh/ElementRegionManager.cpp b/src/coreComponents/mesh/ElementRegionManager.cpp index 7b242bb5a22..89c2985b975 100644 --- a/src/coreComponents/mesh/ElementRegionManager.cpp +++ b/src/coreComponents/mesh/ElementRegionManager.cpp @@ -153,6 +153,15 @@ void ElementRegionManager::generateMesh( CellBlockManagerABC const & cellBlockMa } +void ElementRegionManager::generateMesh( generators::MeshMappings const & meshMappings ) +{ + this->forElementRegions< CellElementRegion >( [&]( CellElementRegion & elemRegion ) + { + elemRegion.generateMesh( meshMappings.getNeighbors(), meshMappings.getCellMgr().getCellBlks() ); + } ); + // TODO finish the work for fractures. +} + void ElementRegionManager::generateWells( CellBlockManagerABC const & cellBlockManager, MeshLevel & meshLevel ) { @@ -682,6 +691,34 @@ ElementRegionManager::getCellBlockToSubRegionMap( CellBlockManagerABC const & ce return blockMap; } +array2d< localIndex > +ElementRegionManager::getCellBlockToSubRegionMap( std::map< string, generators::CellBlk const * > const & cb ) const +{ + array2d< localIndex > blockMap( std::size( cb ), 2 ); + blockMap.setValues< serialPolicy >( -1 ); + + forElementSubRegionsComplete< CellElementSubRegion >( [blockMap = blockMap.toView(), + &cb]( localIndex const er, + localIndex const esr, + ElementRegionBase const & region, + CellElementSubRegion const & subRegion ) + { + GEOS_UNUSED_VAR( region ); // unused if geos_error_if is null + localIndex const blockIndex = std::distance( std::cbegin( cb ), cb.find( subRegion.getName() ) ); + GEOS_ERROR_IF( blockIndex == Group::subGroupMap::KeyIndex::invalid_index, + GEOS_FMT( "{}, subregion {}: Cell block not found at index {}.", + region.getDataContext().toString(), subRegion.getName(), blockIndex ) ); + GEOS_ERROR_IF( blockMap( blockIndex, 1 ) != -1, + GEOS_FMT( "{}, subregion {}: Cell block at index {} is mapped to more than one subregion.", + region.getDataContext().toString(), subRegion.getName(), blockIndex ) ); + + blockMap( blockIndex, 0 ) = er; + blockMap( blockIndex, 1 ) = esr; + } ); + + return blockMap; +} + REGISTER_CATALOG_ENTRY( ObjectManagerBase, ElementRegionManager, string const &, Group * const ) } diff --git a/src/coreComponents/mesh/ElementRegionManager.hpp b/src/coreComponents/mesh/ElementRegionManager.hpp index 9a75cbc164b..fd30fe27331 100644 --- a/src/coreComponents/mesh/ElementRegionManager.hpp +++ b/src/coreComponents/mesh/ElementRegionManager.hpp @@ -23,6 +23,7 @@ #include "CellElementRegion.hpp" #include "CellElementSubRegion.hpp" #include "mesh/generators/CellBlockManagerABC.hpp" +#include "mesh/generators/include/MeshMappings.hpp" #include "mesh/ObjectManagerBase.hpp" #include "dataRepository/ReferenceWrapper.hpp" #include "SurfaceElementRegion.hpp" @@ -148,6 +149,9 @@ class ElementRegionManager : public ObjectManagerBase */ void generateMesh( CellBlockManagerABC const & cellBlockManager ); + + void generateMesh( generators::MeshMappings const & meshMappings ); + /** * @brief Generate the wells. * @param [in] cellBlockManager pointer to cellBlockManager @@ -273,6 +277,7 @@ class ElementRegionManager : public ObjectManagerBase * region (first entry) and subregion (second entry) indices, or -1 if block was not used. */ array2d< localIndex > getCellBlockToSubRegionMap( CellBlockManagerABC const & cellBlockManager ) const; + array2d< localIndex > getCellBlockToSubRegionMap( std::map< string, generators::CellBlk const * > const & cb ) const; /** * @brief This function is used to launch kernel function over all the element regions with region type = diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index 93ab06f4c64..f14139119c7 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -206,6 +206,34 @@ void FaceManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock } } +void FaceManager::setGeometricalRelations( generators::MeshMappings const & meshMappings, + ElementRegionManager const & elemRegionManager, + NodeManager const & nodeManager ) +{ + GEOS_MARK_FUNCTION; + auto const & faceMgr = meshMappings.getFaceMgr(); + + resize( faceMgr.numFaces() ); + + m_toNodesRelation.base() = faceMgr.getFaceToNodes(); + m_toEdgesRelation.base() = faceMgr.getFaceToEdges(); + + // TODO This is new + m_localToGlobalMap = faceMgr.getLocalToGlobal(); + m_globalToLocalMap = faceMgr.getGlobalToLocal(); + + // TODO not for there, but it's convenient + copyExchangeInfo( meshMappings.getNeighbors(), faceMgr.getGhostRank(), faceMgr.getSend(), faceMgr.getRecv() ); + + ToCellRelation< array2d< localIndex > > const toCellBlock = faceMgr.getFaceToElements(); + array2d< localIndex > const blockToSubRegion = elemRegionManager.getCellBlockToSubRegionMap( meshMappings.getCellMgr().getCellBlks() ); // TODO This already exists in NodeManager + meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( blockToSubRegion.toViewConst(), + toCellBlock, + m_toElements ); + // TODO add the fracture stuff + computeGeometry( nodeManager ); +} + void FaceManager::setupRelatedObjectsInRelations( NodeManager const & nodeManager, EdgeManager const & edgeManager, ElementRegionManager const & elemRegionManager ) diff --git a/src/coreComponents/mesh/FaceManager.hpp b/src/coreComponents/mesh/FaceManager.hpp index 2c7948b145f..f0362246062 100644 --- a/src/coreComponents/mesh/FaceManager.hpp +++ b/src/coreComponents/mesh/FaceManager.hpp @@ -20,6 +20,7 @@ #define GEOS_MESH_FACEMANAGER_HPP_ #include "mesh/generators/CellBlockManagerABC.hpp" +#include "mesh/generators/include/MeshMappings.hpp" #include "mesh/ObjectManagerBase.hpp" #include "ToElementRelation.hpp" @@ -123,6 +124,18 @@ class FaceManager : public ObjectManagerBase ElementRegionManager const & elemRegionManager, NodeManager const & nodeManager, bool isBaseMeshLevel ); + /** + * @brief Initialise the current @c FaceManager with the information from @c meshMappings. + * @param[in] meshMappings Provides all the geometrical mappings. + * @param[in] elemRegionManager Provides the mapping from the incoming geometrical cell blocks to their sub-region twin. + * @param[in] nodeManager Provides the positions of the nodes in order to perform the area computations. + * @note We could have relied on the node positions of the @c meshMappings in order to reduce the number of input parameters. + * This is disputable since we do not want to have multiple sources of data which, while currenly identical, may in the future differ. + */ + void setGeometricalRelations( generators::MeshMappings const & meshMappings, + ElementRegionManager const & elemRegionManager, + NodeManager const & nodeManager ); + /** * @brief Link the current manager to other managers. * @param[in] nodeManager The node manager instance. diff --git a/src/coreComponents/mesh/NodeManager.cpp b/src/coreComponents/mesh/NodeManager.cpp index 2bc39dc498f..116d26e5301 100644 --- a/src/coreComponents/mesh/NodeManager.cpp +++ b/src/coreComponents/mesh/NodeManager.cpp @@ -193,6 +193,36 @@ void NodeManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock elemRegionManager.forElementRegionsComplete< SurfaceElementRegion >( connectNodesTo2dElements ); } +void NodeManager::setGeometricalRelations( generators::MeshMappings const & meshMappings, + ElementRegionManager const & elemRegionManager ) +{ + GEOS_MARK_FUNCTION; + auto const & nodeMgr = meshMappings.getNodeMgr(); + + resize( nodeMgr.numNodes() ); + + m_referencePosition = nodeMgr.getNodePositions(); + + // TODO I add the copy of the g2l and l2g mappings here. This is new + m_localToGlobalMap = nodeMgr.getLocalToGlobal(); + m_globalToLocalMap = nodeMgr.getGlobalToLocal(); + + // TODO not for there, but it's convenient + copyExchangeInfo( meshMappings.getNeighbors(), nodeMgr.getGhostRank(), nodeMgr.getSend(), nodeMgr.getRecv() ); + + m_toEdgesRelation.base().assimilate< parallelHostPolicy >( nodeMgr.getNodeToEdges(), + LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); + m_toFacesRelation.base().assimilate< parallelHostPolicy >( nodeMgr.getNodeToFaces(), + LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); + + ToCellRelation< ArrayOfArrays< localIndex > > const toCellBlock = nodeMgr.getNodeToElements(); + array2d< localIndex > const blockToSubRegion = elemRegionManager.getCellBlockToSubRegionMap( meshMappings.getCellMgr().getCellBlks() ); // TODO This already exists in FaceManager + meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( blockToSubRegion.toViewConst(), + toCellBlock, + m_toElements ); + // TODO add the fracture component +} + void NodeManager::setupRelatedObjectsInRelations( EdgeManager const & edgeManager, FaceManager const & faceManager, ElementRegionManager const & elemRegionManager ) diff --git a/src/coreComponents/mesh/NodeManager.hpp b/src/coreComponents/mesh/NodeManager.hpp index 71c7439fca0..c9e9027ec84 100644 --- a/src/coreComponents/mesh/NodeManager.hpp +++ b/src/coreComponents/mesh/NodeManager.hpp @@ -20,6 +20,7 @@ #define GEOS_MESH_NODEMANAGER_HPP_ #include "mesh/generators/CellBlockManagerABC.hpp" +#include "mesh/generators/include/MeshMappings.hpp" #include "mesh/ObjectManagerBase.hpp" #include "mesh/simpleGeometricObjects/GeometricObjectManager.hpp" #include "ToElementRelation.hpp" @@ -165,6 +166,14 @@ class NodeManager : public ObjectManagerBase ElementRegionManager const & elemRegionManager, bool isBaseMeshLevel ); + /** + * @brief Initialise the current @c NodeManager with the information from @c meshMappings. + * @param[in] meshMappings Provides all the geometrical mappings. + * @param[in] elemRegionManager Provides the mapping from the incoming geometrical cell blocks to their sub-region twin. + */ + void setGeometricalRelations( generators::MeshMappings const & meshMappings, + ElementRegionManager const & elemRegionManager ); + /** * @brief Link the current manager to other managers. * @param edgeManager The edge manager instance. diff --git a/src/coreComponents/mesh/ObjectManagerBase.cpp b/src/coreComponents/mesh/ObjectManagerBase.cpp index 074176c75fa..40d56597c3a 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.cpp +++ b/src/coreComponents/mesh/ObjectManagerBase.cpp @@ -1043,5 +1043,25 @@ void ObjectManagerBase::moveSets( LvArray::MemorySpace const targetSpace ) } ); } +void ObjectManagerBase::copyExchangeInfo( std::set< integer > const & neighbors, + array1d< integer > const & ghostRank, + std::map< integer, array1d< localIndex > > const & send, + std::map< integer, array1d< localIndex > > const & recv ) +{ + m_ghostRank = ghostRank; + for( integer const & rank: neighbors ) + { + addNeighbor( rank ); + } + for( auto const & [rank, lis]: send ) + { + m_neighborData.at( rank ).ghostsToSend() = lis; + } + for( auto const & [rank, lis]: recv) + { + m_neighborData.at( rank ).ghostsToReceive() = lis; + } +} + } /* namespace geos */ diff --git a/src/coreComponents/mesh/ObjectManagerBase.hpp b/src/coreComponents/mesh/ObjectManagerBase.hpp index a85e87afa3f..552c7168495 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.hpp +++ b/src/coreComponents/mesh/ObjectManagerBase.hpp @@ -924,6 +924,17 @@ class ObjectManagerBase : public dataRepository::Group } protected: + /** + * @brief Copy the ghosting exchange information for the entities (e.g. nodes, edges...) held by the current manager. + * @param[in] neighbors The ranks of the neighbors (does not include the current MPI rank) + * @param[in] ghostRank The ghost ranks of the entities own by the current manager. + * @param[in] send The entities to send to which neighbors. + * @param[in] recv The entities that will be received from which neighbor. + */ + void copyExchangeInfo( std::set< integer > const & neighbors, + array1d< integer > const & ghostRank, + std::map< integer, array1d< localIndex > > const & send, + std::map< integer, array1d< localIndex > > const & recv ); /// Group that holds object sets. Group m_sets; diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 999dba87e13..8edfbcd8b1d 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -18,7 +18,7 @@ #include "NewGlobalNumbering.hpp" -//#include "common/MpiWrapper.hpp" +#include "common/MpiWrapper.hpp" #include "codingUtilities/Utilities.hpp" #include "LvArray/src/ArrayOfArraysView.hpp" @@ -161,16 +161,21 @@ ArrayOfArrays< localIndex > convertToAoA( std::map< T, std::vector< U > > const return t2u_; } -template< class T, class U > -array2d< localIndex > convertToA2d( std::map< T, std::vector< U > > const & t2u, - int dimU ) +template< class T, class U, class P=camp::make_idx_seq_t< 2 > > +array2d< localIndex, P > convertToA2d( std::map< T, std::vector< U > > const & t2u, + int dimU, + bool check = true ) { static_assert( !std::is_same_v< T, U > ); - array2d< localIndex > t2u_( std::size( t2u ), dimU ); - t2u_.setValues< serialPolicy >( -1 ); + array2d< localIndex, P > t2u_( std::size( t2u ), dimU ); + t2u_.template setValues< serialPolicy >( -1 ); for( auto const & [t, us]: t2u ) { + if( check ) + { + GEOS_ASSERT_EQ( intConv< int >( std::size( us ) ), dimU ); + } for( int i = 0; i < intConv< int >( std::size( us ) ); ++i ) { t2u_( t.get(), i ) = us[i].get(); @@ -185,6 +190,7 @@ convertGlbLoc( std::map< GI, LI > const & g2l ) { static_assert( !std::is_same_v< GI, LI > ); array1d< globalIndex > l2g_( std::size( g2l ) ); + l2g_.setValues< serialPolicy >( -1 ); unordered_map< globalIndex, localIndex > g2l_; for( auto const & [gi, li]: g2l ) { @@ -195,12 +201,31 @@ convertGlbLoc( std::map< GI, LI > const & g2l ) return { l2g_, g2l_ }; } +template< typename LI > +std::map< integer, array1d< localIndex > > toFlavorlessMapping( std::map< MpiRank, std::vector< LI > > const & input ) +{ + std::map< integer, array1d< localIndex > > output; + for( auto const & [rank, lis]: input ) + { + array1d< localIndex > & tmp = output[rank.get()]; + tmp.reserve( std::size( lis ) ); + for( LI const & li: lis ) + { + tmp.emplace_back( li.get() ); + } + } + + return output; +}; + EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::vector< integer > const & ghostRank, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, std::map< EdgeLocIdx, std::vector< FaceLocIdx > > const & e2f, std::map< EdgeGlbIdx, EdgeLocIdx > const & eg2l, - std::map< EdgeLocIdx, EdgeGlbIdx > const & el2g ) + std::map< EdgeLocIdx, EdgeGlbIdx > const & el2g, + std::map< MpiRank, std::vector< EdgeLocIdx > > const & send, + std::map< MpiRank, std::vector< EdgeLocIdx > > const & recv ) { GEOS_ASSERT_EQ( numEdges, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numEdges, std::size( e2n ) ); @@ -218,13 +243,21 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, auto [l2g, g2l] = convertGlbLoc( eg2l ); - array1d< integer > ghostRank_( numEdges ); + array1d< integer > ghostRank_; + ghostRank_.reserve( numEdges ); for( integer const & gr: ghostRank ) { ghostRank_.emplace_back( gr ); } - return EdgeMgrImpl( numEdges, std::move( ghostRank_ ), std::move( e2n_ ), convertToAoA( e2f ), std::move( g2l ), std::move( l2g ) ); + return EdgeMgrImpl( numEdges, + std::move( ghostRank_ ), + std::move( e2n_ ), + convertToAoA( e2f ), + std::move( g2l ), + std::move( l2g ), + toFlavorlessMapping( send ), + toFlavorlessMapping( recv ) ); } FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, @@ -233,7 +266,9 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const & f2e, std::map< FaceLocIdx, std::vector< CellLocIdx > > const & f2c, std::map< FaceGlbIdx, FaceLocIdx > const & fg2l, - std::map< FaceLocIdx, FaceGlbIdx > const & fl2g ) + std::map< FaceLocIdx, FaceGlbIdx > const & fl2g, + std::map< MpiRank, std::vector< FaceLocIdx > > const & send, + std::map< MpiRank, std::vector< FaceLocIdx > > const & recv ) { GEOS_ASSERT_EQ( numFaces, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numFaces, std::size( f2n ) ); @@ -244,13 +279,21 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, auto [l2g, g2l] = convertGlbLoc( fg2l ); - array1d< integer > ghostRank_( numFaces ); + array1d< integer > ghostRank_; + ghostRank_.reserve( numFaces ); for( integer const & gr: ghostRank ) { ghostRank_.emplace_back( gr ); } - return FaceMgrImpl( numFaces, std::move( ghostRank_ ), convertToAoA( f2n ), convertToAoA( f2e ), convertToA2d( f2c, 2 ), std::move( g2l ), std::move( l2g ) ); + return FaceMgrImpl( numFaces, std::move( ghostRank_ ), + convertToAoA( f2n ), + convertToAoA( f2e ), + convertToA2d( f2c, 2, false ), + std::move( g2l ), + std::move( l2g ), + toFlavorlessMapping( send ), + toFlavorlessMapping( recv ) ); } NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, @@ -260,7 +303,9 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, std::map< NodeLocIdx, std::vector< FaceLocIdx > > const & n2f, std::map< NodeLocIdx, std::vector< CellLocIdx > > const & n2c, std::map< NodeGlbIdx, NodeLocIdx > const & ng2l, - std::map< NodeLocIdx, NodeGlbIdx > const & nl2g ) + std::map< NodeLocIdx, NodeGlbIdx > const & nl2g, + std::map< MpiRank, std::vector< NodeLocIdx > > const & send, + std::map< MpiRank, std::vector< NodeLocIdx > > const & recv ) { GEOS_ASSERT_EQ( numNodes, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numNodes, std::size( n2pos ) ); @@ -274,7 +319,8 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock auto [l2g, g2l] = convertGlbLoc( ng2l ); - array1d< integer > ghostRank_( numNodes ); + array1d< integer > ghostRank_; + ghostRank_.reserve( numNodes ); for( integer const & gr: ghostRank ) { ghostRank_.emplace_back( gr ); @@ -284,12 +330,22 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, for( auto const & [ngi, pos]: n2pos ) { NodeLocIdx const nli = ng2l.at( ngi ); - positions( nli.get(), 0 ) = pos[0]; - positions( nli.get(), 1 ) = pos[2]; - positions( nli.get(), 2 ) = pos[2]; + for( auto i = 0; i < 3; ++i ) + { + positions( nli.get(), i ) = pos[i]; + } } - return NodeMgrImpl( intConv< localIndex >( numNodes ), std::move( positions ), std::move( ghostRank_ ), convertToAoA( n2e ), convertToAoA( n2f ), convertToAoA( n2c ), std::move( l2g ) ); + return NodeMgrImpl( intConv< localIndex >( numNodes ), + std::move( positions ), + std::move( ghostRank_ ), + convertToAoA( n2e ), + convertToAoA( n2f ), + convertToAoA( n2c ), + std::move( l2g ), + std::move( g2l ), + toFlavorlessMapping( send ), + toFlavorlessMapping( recv ) ); } CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, @@ -298,7 +354,9 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, std::map< CellGlbIdx, CellLocIdx > const & cg2l, - std::map< CellLocIdx, CellGlbIdx > const & cl2g ) + std::map< CellLocIdx, CellGlbIdx > const & cl2g, + std::map< MpiRank, std::vector< CellLocIdx > > const & send, + std::map< MpiRank, std::vector< CellLocIdx > > const & recv ) { GEOS_ASSERT_EQ( numCells, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numCells, std::size( c2n ) ); @@ -311,13 +369,22 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock auto [l2g, g2l] = convertGlbLoc( cg2l ); - array1d< integer > ghostRank_( numCells ); + array1d< integer > ghostRank_; + ghostRank_.reserve( numCells ); for( integer const & gr: ghostRank ) { ghostRank_.emplace_back( gr ); } - return CellBlkImpl( intConv< localIndex >( numCells ), ghostRank_, convertToA2d( c2n, 8 ), convertToA2d( c2e, 12 ), convertToA2d( c2f, 6 ), std::move( l2g ) ); + return CellBlkImpl( intConv< localIndex >( numCells ), + std::move( ghostRank_ ), + convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, 8 ), + convertToA2d( c2e, 12 ), + convertToA2d( c2f, 6 ), + std::move( l2g ), + std::move( g2l ), + toFlavorlessMapping( send ), + toFlavorlessMapping( recv ) ); } DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, @@ -390,9 +457,8 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); // TODO carefully check the ordering... - res.c2n[cli] = { bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; -// nodes.insert( std::end( nodes ), std::cbegin( bn ), std::cend( bn ) ); -// nodes.insert( std::end( nodes ), std::cbegin( tn ), std::cend( tn ) ); + std::array< NodeLocIdx, 8 > const tmp{ bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; + res.c2n[cli] = { tmp[0], tmp[1], tmp[3], tmp[2], tmp[4], tmp[5], tmp[7], tmp[6] }; } return res; @@ -453,6 +519,85 @@ UpwardMappings buildUpwardMappings( DownwardMappings const & downwardMappings ) return res; } +std::set< integer > getNeighbors( GhostRecv const & recv, + GhostSend const & send ) +{ + std::set< MpiRank > ranks; + + auto const insertSingle = [&]( auto const & geom2neighbor ) + { + for( auto const & [_, neighbor]: geom2neighbor ) + { + ranks.insert( neighbor ); + } + }; + + auto const insertMultiple = [&]( auto const & geom2neighbors ) + { + for( auto const & [_, neighbors]: geom2neighbors ) + { + ranks.insert( std::cbegin( neighbors ), std::cend( neighbors ) ); + } + }; + + insertSingle( recv.nodes ); + insertSingle( recv.edges ); + insertSingle( recv.faces ); + insertSingle( recv.cells ); + + insertMultiple( send.nodes ); + insertMultiple( send.edges ); + insertMultiple( send.faces ); + insertMultiple( send.cells ); + + std::set< integer > result; + for( MpiRank const & rank: ranks ) + { + result.insert( rank.get() ); + } + + return result; +} + +template< typename GI, typename LI > +std::map< MpiRank, std::vector< LI > > invertGhostSend( std::map< GI, std::set< MpiRank > > const & input, + std::map< GI, LI > const & g2l ) +{ + std::map< MpiRank, std::vector< LI > > output; + for( auto const & [gi, ranks]: input ) + { + for( MpiRank const & rank: ranks ) + { + output[rank].emplace_back( g2l.at( gi ) ); + } + } + + for( auto & [_, lis]: output ) + { + std::sort( std::begin( lis ), std::end( lis ) ); + } + + return output; +} + +template< typename GI, typename LI > +std::map< MpiRank, std::vector< LI > > invertGhostRecv( std::map< GI, MpiRank > const & input, + std::map< GI, LI > const & g2l ) +{ + std::map< MpiRank, std::vector< LI > > output; + for( auto const & [gi, rank]: input ) + { + output[rank].emplace_back( g2l.at( gi ) ); + } + + for( auto & [_, lis]: output ) + { + std::sort( std::begin( lis ), std::end( lis ) ); + } + + return output; +} + void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, @@ -467,44 +612,54 @@ void buildPods( MeshGraph const & owned, DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); - NodeMgrImpl const nodeMgr = makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), - buildGhostRank( g2l.nodes, send.nodes, recv.nodes ), - graph.n2pos, - upwardMappings.n2e, - upwardMappings.n2f, - upwardMappings.n2c, - g2l.nodes, - l2g.nodes ); - - EdgeMgrImpl const edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), - buildGhostRank( g2l.edges, send.edges, recv.edges ), - downwardMappings.e2n, - upwardMappings.e2f, - g2l.edges, - l2g.edges ); - - FaceMgrImpl const faceMgr = makeFlavorlessFaceMgrImpl( std::size( g2l.faces ), - buildGhostRank( g2l.faces, send.faces, recv.faces ), - downwardMappings.f2n, - downwardMappings.f2e, - upwardMappings.f2c, - g2l.faces, - l2g.faces ); - - CellBlkImpl const cellBlock = makeFlavorlessCellBlkImpl( std::size( g2l.cells ), - buildGhostRank( g2l.cells, send.cells, recv.cells ), - downwardMappings.c2n, - downwardMappings.c2e, - downwardMappings.c2f, - g2l.cells, - l2g.cells ); - - CellMgrImpl const cellMgr( cellBlock ); - - meshMappings.setCellMgr( cellMgr ); - meshMappings.setEdgeMgr( edgeMgr ); - meshMappings.setFaceMgr( faceMgr ); - meshMappings.setNodeMgr( nodeMgr ); + NodeMgrImpl nodeMgr = makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), + buildGhostRank( g2l.nodes, send.nodes, recv.nodes ), + graph.n2pos, + upwardMappings.n2e, + upwardMappings.n2f, + upwardMappings.n2c, + g2l.nodes, + l2g.nodes, + invertGhostSend( send.nodes, g2l.nodes ), + invertGhostRecv( recv.nodes, g2l.nodes ) ); + + EdgeMgrImpl edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), + buildGhostRank( g2l.edges, send.edges, recv.edges ), + downwardMappings.e2n, + upwardMappings.e2f, + g2l.edges, + l2g.edges, + invertGhostSend( send.edges, g2l.edges ), + invertGhostRecv( recv.edges, g2l.edges ) ); + + FaceMgrImpl faceMgr = makeFlavorlessFaceMgrImpl( std::size( g2l.faces ), + buildGhostRank( g2l.faces, send.faces, recv.faces ), + downwardMappings.f2n, + downwardMappings.f2e, + upwardMappings.f2c, + g2l.faces, + l2g.faces, + invertGhostSend( send.faces, g2l.faces ), + invertGhostRecv( recv.faces, g2l.faces ) ); + + CellBlkImpl cellBlock = makeFlavorlessCellBlkImpl( std::size( g2l.cells ), + buildGhostRank( g2l.cells, send.cells, recv.cells ), + downwardMappings.c2n, + downwardMappings.c2e, + downwardMappings.c2f, + g2l.cells, + l2g.cells, + invertGhostSend( send.cells, g2l.cells ), + invertGhostRecv( recv.cells, g2l.cells ) ); + + CellMgrImpl cellMgr( std::move( cellBlock ) ); + + meshMappings.setCellMgr( std::move( cellMgr ) ); + meshMappings.setEdgeMgr( std::move( edgeMgr ) ); + meshMappings.setFaceMgr( std::move( faceMgr ) ); + meshMappings.setNodeMgr( std::move( nodeMgr ) ); + + meshMappings.setNeighbors( getNeighbors( recv, send ) ); } } diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index f522332958d..fda8930c14e 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -32,7 +32,7 @@ namespace geos::ghosting * @details When a cell refers to a face, w.r.t. the canonical ordering of the face, * the nodes of the face (in the cell) can be shifted, * and travelled in a different direction (clockwise or counter-clockwise). - * The @c isFlipped parameter informs about the travel discrepancy. + * The @c isFlipped parameter informs about the travel direction. * The @c start parameter informs about the shift. * @note This class does not refer to how multiple faces are ordered by a cell, * but to the precise information when refering to one given face. diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 8578d395b09..d5a202a164c 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -731,9 +731,9 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // the number of columns being the number of nodes in the mesh graph, // the number of rows being the number of MPI ranks. // - // As the result of the multiplication of the `ghostingFootprint` matrix and the `ownership`, + // As the result of the multiplication between the `ghostingFootprint` matrix and the `ownership` matrix, // for each row owned (ie at the current MPI rank index), - // the value of the matrix term will provide the actual owning rank for all the . + // the value of the `ghostExchange` matrix term will provide the actual owning rank for all the . // // From `ghostExchange` we can extract which other rank will send to the current rank any graph node. Epetra_CrsMatrix ghostExchange( Epetra_DataAccess::Copy, mpiMap, 1, false ); @@ -762,27 +762,41 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & ghostingFootprint.ExtractGlobalRowCopy( index, length, extracted, extractedValues.data(), extractedIndices.data() ); GEOS_ASSERT_EQ( extracted, length ); - std::set< MpiRank > * sendingTo = nullptr; + std::set< MpiRank > neighbors; + for( int i = 0; i < extracted; ++i ) + { + MpiRank const rank{ extractedIndices[i] }; + if( rank != curRank ) + { + neighbors.insert( rank ); + } + } + + if( std::empty( neighbors ) ) + { + continue; + } + switch( convert.getGeometricalType( index ) ) { case Geom::NODE: { - sendingTo = &send.nodes[convert.toNodeGlbIdx( index )]; + send.nodes.emplace( convert.toNodeGlbIdx( index ), std::move( neighbors ) ); break; } case Geom::EDGE: { - sendingTo = &send.edges[convert.toEdgeGlbIdx( index )]; + send.edges.emplace( convert.toEdgeGlbIdx( index ), std::move( neighbors ) ); break; } case Geom::FACE: { - sendingTo = &send.faces[convert.toFaceGlbIdx( index )]; + send.faces.emplace( convert.toFaceGlbIdx( index ), std::move( neighbors ) ); break; } case Geom::CELL: { - sendingTo = &send.cells[convert.toCellGlbIdx( index )]; + send.cells.emplace( convert.toCellGlbIdx( index ), std::move( neighbors ) ); break; } default: @@ -790,15 +804,6 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & GEOS_ERROR( "Internal error" ); } } - - for( int ii = 0; ii < extracted; ++ii ) - { - MpiRank const rank{ extractedIndices[ii] }; - if( rank != curRank ) - { - sendingTo->insert( rank ); - } - } } int const numNeededIndices = ghostExchange.NumGlobalEntries( curRank.get() ); diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 713afc5809b..f553a12aeb7 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -23,36 +23,18 @@ NodeMgrImpl::NodeMgrImpl( localIndex numNodes, ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, - array1d< globalIndex > const & l2g ) - : m_numNodes( numNodes ), + array1d< globalIndex > && l2g, + unordered_map< globalIndex, localIndex > && g2l, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ) + : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + m_numNodes( numNodes ), m_positions( positions ), - m_ghostRank( ghostRank ), m_n2e( n2e ), m_n2f( n2f ), - m_n2c( n2c ), - m_l2g( l2g ) + m_n2c( n2c ) { } -localIndex NodeMgrImpl::numNodes() const -{ - return m_numNodes; -} - -array2d< real64, nodes::REFERENCE_POSITION_PERM > NodeMgrImpl::getNodePositions() const -{ - return m_positions; -} - -ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToEdges() const -{ - return m_n2e; -} - -ArrayOfArrays< localIndex > NodeMgrImpl::getNodeToFaces() const -{ - return m_n2f; -} - ToCellRelation< ArrayOfArrays< localIndex > > NodeMgrImpl::getNodeToElements() const { ArrayOfArrays< localIndex > toBlockIndex( m_n2c ); // TODO cheat in case there's one unique cell block! @@ -69,86 +51,36 @@ ToCellRelation< ArrayOfArrays< localIndex > > NodeMgrImpl::getNodeToElements() c return { std::move( toBlockIndex ), m_n2c }; } -array1d< globalIndex > NodeMgrImpl::getLocalToGlobal() const -{ - return m_l2g; -} - -std::map< string, SortedArray< localIndex > > const & NodeMgrImpl::getNodeSets() const -{ - return m_todo; -} - EdgeMgrImpl::EdgeMgrImpl( std::size_t numEdges, array1d< integer > && ghostRank, array2d< localIndex > && e2n, ArrayOfArrays< localIndex > && e2f, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g ) - : m_numEdges( numEdges ), - m_ghostRank( ghostRank ), + array1d< globalIndex > && l2g, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ) + : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + m_numEdges( numEdges ), m_e2n( e2n ), - m_e2f( e2f ), - m_g2l( g2l ), - m_l2g( l2g ) + m_e2f( e2f ) { } -localIndex EdgeMgrImpl::numEdges() const -{ - return m_numEdges; -} - -array2d< localIndex > EdgeMgrImpl::getEdgeToNodes() const -{ - return m_e2n; -} - -ArrayOfArrays< localIndex > EdgeMgrImpl::getEdgeToFaces() const -{ - return m_e2f; -} - -array1d< integer > EdgeMgrImpl::getGhostRank() const -{ - return m_ghostRank; -} - -array1d< globalIndex > EdgeMgrImpl::getLocalToGlobal() const -{ - return m_l2g; -} - FaceMgrImpl::FaceMgrImpl( std::size_t numFaces, array1d< integer > && ghostRank, ArrayOfArrays< localIndex > && f2n, ArrayOfArrays< localIndex > && f2e, array2d< localIndex > && f2c, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g ) - : m_numFaces( numFaces ), - m_ghostRank( ghostRank ), + array1d< globalIndex > && l2g, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ) + : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + m_numFaces( numFaces ), m_f2n( f2n ), m_f2e( f2e ), - m_f2c( f2c ), - m_g2l( g2l ), - m_l2g( l2g ) + m_f2c( f2c ) { } -localIndex FaceMgrImpl::numFaces() const -{ - return intConv< localIndex >( m_numFaces ); -} - -ArrayOfArrays< localIndex > FaceMgrImpl::getFaceToNodes() const -{ - return m_f2n; -} - -ArrayOfArrays< localIndex > FaceMgrImpl::getFaceToEdges() const -{ - return m_f2e; -} - ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const { array2d< localIndex > toBlockIndex( m_f2c ); // TODO cheat in case there's one unique cell block! @@ -165,28 +97,20 @@ ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const return { std::move( toBlockIndex ), m_f2c }; } -array1d< integer > FaceMgrImpl::getGhostRank() const -{ - return m_ghostRank; -} - -array1d< globalIndex > FaceMgrImpl::getLocalToGlobal() const -{ - return m_l2g; -} - CellBlkImpl::CellBlkImpl( localIndex numCells, - array1d< integer > const & ghostRank, + array1d< integer > && ghostRank, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, - array1d< globalIndex > const & l2g ) - : m_numCells( numCells ), - m_ghostRank( ghostRank ), + array1d< globalIndex > && l2g, + unordered_map< globalIndex, localIndex > && g2l, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ) + : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + m_numCells( numCells ), m_c2n( c2n ), m_c2e( c2e ), - m_c2f( c2f ), - m_l2g( l2g ) + m_c2f( c2f ) { } ElementType CellBlkImpl::getElementType() const @@ -209,59 +133,9 @@ localIndex CellBlkImpl::numFacesPerElement() const return 6; } -localIndex CellBlkImpl::numElements() const -{ - return m_numCells; -} - -array2d< localIndex, cells::NODE_MAP_PERMUTATION > CellBlkImpl::getElemToNodes() const -{ - return m_c2n; -} - -array2d< localIndex > CellBlkImpl::getElemToEdges() const -{ - return m_c2e; -} - -array2d< localIndex > CellBlkImpl::getElemToFaces() const -{ - return m_c2f; -} - -array1d< globalIndex > CellBlkImpl::localToGlobalMap() const -{ - return m_l2g; -} - -//std::list< dataRepository::WrapperBase const * > CellBlkImpl::getExternalProperties() const -//{ -// return {}; -//} - -generators::CellMgr const & MeshMappingImpl::getCellMgr() const -{ - return m_cellMgr; -} - -generators::EdgeMgr const & MeshMappingImpl::getEdgeMgr() const -{ - return m_edgeMgr; -} - -generators::FaceMgr const & MeshMappingImpl::getFaceMgr() const -{ - return m_faceMgr; -} - -generators::NodeMgr const & MeshMappingImpl::getNodeMgr() const -{ - return m_nodeMgr; -} - -std::list< CellBlk const * > CellMgrImpl::getCellBlks() const +std::map< string, generators::CellBlk const * > CellMgrImpl::getCellBlks() const { - return { &m_cellBlk }; + return { { string( "hexahedra" ), &m_cellBlk } }; // TODO hard coded values. } } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 1747c94103e..12759c37023 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -17,6 +17,7 @@ #include "Indices.hpp" +#include "include/GhostExchange.hpp" #include "include/NodeMgr.hpp" #include "include/EdgeMgr.hpp" #include "include/FaceMgr.hpp" @@ -27,11 +28,19 @@ namespace geos { +struct GhostMapping +{ + array1d< integer > m_ghostRank; + array1d< globalIndex > m_l2g; + unordered_map< globalIndex, localIndex > m_g2l; + std::map< integer, array1d< localIndex > > m_send; + std::map< integer, array1d< localIndex > > m_recv; +}; + class NodeMgrImpl : public generators::NodeMgr { public: - NodeMgrImpl() - { } + NodeMgrImpl() = default; NodeMgrImpl( localIndex numNodes, array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, @@ -39,70 +48,139 @@ class NodeMgrImpl : public generators::NodeMgr ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, - array1d< globalIndex > const & l2g ); + array1d< globalIndex > && l2g, + unordered_map< globalIndex, localIndex > && g2l, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ); + + [[nodiscard]] localIndex numNodes() const override + { + return m_numNodes; + } + + [[nodiscard]] array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override + { + return m_positions; + } + + [[nodiscard]] ArrayOfArrays< localIndex > getNodeToEdges() const override + { + return m_n2e; + } - [[nodiscard]] localIndex numNodes() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getNodeToFaces() const override + { + return m_n2f; + } - [[nodiscard]] array2d< real64, nodes::REFERENCE_POSITION_PERM > getNodePositions() const override; + [[nodiscard]] ToCellRelation< ArrayOfArrays< localIndex > > getNodeToElements() const override; - [[nodiscard]] ArrayOfArrays< localIndex > getNodeToEdges() const override; + [[nodiscard]] std::map< string, SortedArray< localIndex > > const & getNodeSets() const override + { + return m_todo; + } - [[nodiscard]] ArrayOfArrays< localIndex > getNodeToFaces() const override; + // Diamond + [[nodiscard]] array1d< integer > getGhostRank() const override + { + return m_ghost.m_ghostRank; + } - [[nodiscard]] ToCellRelation< ArrayOfArrays< localIndex>> getNodeToElements() const override; + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override + { + return m_ghost.m_l2g; + } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; + [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override + { + return m_ghost.m_g2l; + } - [[nodiscard]] std::map< string, SortedArray< localIndex > > const & getNodeSets() const override; + [[nodiscard]] std::map< integer, array1d< localIndex > > getSend() const override + { + return m_ghost.m_send; + } + [[nodiscard]] std::map< integer, array1d< localIndex > > getRecv() const override + { + return m_ghost.m_recv; + } private: + GhostMapping m_ghost; // Diamond localIndex m_numNodes; array2d< real64, nodes::REFERENCE_POSITION_PERM > m_positions; - array1d< integer > m_ghostRank; ArrayOfArrays< localIndex > m_n2e; ArrayOfArrays< localIndex > m_n2f; ArrayOfArrays< localIndex > m_n2c; - array1d< globalIndex > m_l2g; std::map< string, SortedArray< localIndex > > m_todo; }; class EdgeMgrImpl : public generators::EdgeMgr { public: - EdgeMgrImpl() - { } + EdgeMgrImpl() = default; EdgeMgrImpl( std::size_t numEdges, array1d< integer > && ghostRank, array2d< localIndex > && e2n, ArrayOfArrays< localIndex > && e2f, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g ); + array1d< globalIndex > && l2g, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ); - [[nodiscard]] localIndex numEdges() const override; + [[nodiscard]] localIndex numEdges() const override + { + return m_numEdges; + } + + [[nodiscard]] array2d< localIndex > getEdgeToNodes() const override + { + return m_e2n; + } + + [[nodiscard]] ArrayOfArrays< localIndex > getEdgeToFaces() const override + { + return m_e2f; + } + + // Diamond + [[nodiscard]] array1d< integer > getGhostRank() const override + { + return m_ghost.m_ghostRank; + } - [[nodiscard]] array2d< localIndex > getEdgeToNodes() const override; + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override + { + return m_ghost.m_l2g; + } - [[nodiscard]] ArrayOfArrays< localIndex > getEdgeToFaces() const override; + [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override + { + return m_ghost.m_g2l; + } - [[nodiscard]] array1d< integer > getGhostRank() const override; + [[nodiscard]] std::map< integer, array1d< localIndex > > getSend() const override + { + return m_ghost.m_send; + } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; + [[nodiscard]] std::map< integer, array1d< localIndex > > getRecv() const override + { + return m_ghost.m_recv; + } private: + GhostMapping m_ghost; // Diamond localIndex m_numEdges; - array1d< integer > m_ghostRank; array2d< localIndex > m_e2n; ArrayOfArrays< localIndex > m_e2f; - unordered_map< globalIndex, localIndex > m_g2l; - array1d< globalIndex > m_l2g; }; class FaceMgrImpl : public generators::FaceMgr { public: - FaceMgrImpl() - { } + FaceMgrImpl() = default; FaceMgrImpl( std::size_t numFaces, array1d< integer > && ghostRank, @@ -110,42 +188,74 @@ class FaceMgrImpl : public generators::FaceMgr ArrayOfArrays< localIndex > && f2e, array2d< localIndex > && f2c, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g ); + array1d< globalIndex > && l2g, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ); - [[nodiscard]] localIndex numFaces() const override; + [[nodiscard]] localIndex numFaces() const override + { + return intConv< localIndex >( m_numFaces ); + } - [[nodiscard]] ArrayOfArrays< localIndex > getFaceToNodes() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getFaceToNodes() const override + { + return m_f2n; + } - [[nodiscard]] ArrayOfArrays< localIndex > getFaceToEdges() const override; + [[nodiscard]] ArrayOfArrays< localIndex > getFaceToEdges() const override + { + return m_f2e; + } [[nodiscard]] ToCellRelation< array2d< localIndex > > getFaceToElements() const override; - [[nodiscard]] array1d< integer > getGhostRank() const override; + // Diamond + [[nodiscard]] array1d< integer > getGhostRank() const override + { + return m_ghost.m_ghostRank; + } + + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override + { + return m_ghost.m_l2g; + } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override; + [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override + { + return m_ghost.m_g2l; + } + [[nodiscard]] std::map< integer, array1d< localIndex > > getSend() const override + { + return m_ghost.m_send; + } + + [[nodiscard]] std::map< integer, array1d< localIndex > > getRecv() const override + { + return m_ghost.m_recv; + } private: + GhostMapping m_ghost; // Diamond localIndex m_numFaces; - array1d< integer > m_ghostRank; ArrayOfArrays< localIndex > m_f2n; ArrayOfArrays< localIndex > m_f2e; array2d< localIndex > m_f2c; - unordered_map< globalIndex, localIndex > m_g2l; - array1d< globalIndex > m_l2g; }; -class CellBlkImpl : public CellBlk +class CellBlkImpl : public generators::CellBlk { public: - CellBlkImpl() - { } + CellBlkImpl() = default; CellBlkImpl( localIndex numCells, - array1d< integer > const & ghostRank, + array1d< integer > && ghostRank, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, - array1d< globalIndex > const & l2g ); + array1d< globalIndex > && l2g, + unordered_map< globalIndex, localIndex > && m_g2l, + std::map< integer, array1d< localIndex > > && send, + std::map< integer, array1d< localIndex > > && recv ); [[nodiscard]] ElementType getElementType() const override; @@ -155,37 +265,70 @@ class CellBlkImpl : public CellBlk [[nodiscard]] localIndex numFacesPerElement() const override; - [[nodiscard]] localIndex numElements() const override; + [[nodiscard]] localIndex numElements() const override + { + return m_numCells; + } - [[nodiscard]] array2d< localIndex, cells::NODE_MAP_PERMUTATION > getElemToNodes() const override; + [[nodiscard]] array2d< localIndex, cells::NODE_MAP_PERMUTATION > getElemToNodes() const override + { + return m_c2n; + } - [[nodiscard]] array2d< localIndex > getElemToEdges() const override; + [[nodiscard]] array2d< localIndex > getElemToEdges() const override + { + return m_c2e; + } - [[nodiscard]] array2d< localIndex > getElemToFaces() const override; + [[nodiscard]] array2d< localIndex > getElemToFaces() const override + { + return m_c2f; + } - [[nodiscard]] array1d< globalIndex > localToGlobalMap() const override; + // Diamond + [[nodiscard]] array1d< integer > getGhostRank() const override + { + return m_ghost.m_ghostRank; + } + + [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override + { + return m_ghost.m_l2g; + } + + [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override + { + return m_ghost.m_g2l; + } + [[nodiscard]] std::map< integer, array1d< localIndex > > getSend() const override + { + return m_ghost.m_send; + } + + [[nodiscard]] std::map< integer, array1d< localIndex > > getRecv() const override + { + return m_ghost.m_recv; + } private: + GhostMapping m_ghost; // Diamond localIndex m_numCells; - array1d< integer > m_ghostRank; array2d< localIndex, cells::NODE_MAP_PERMUTATION > m_c2n; array2d< localIndex > m_c2e; array2d< localIndex > m_c2f; - array1d< globalIndex > m_l2g; }; class CellMgrImpl : public generators::CellMgr { public: - CellMgrImpl() - { } + CellMgrImpl() = default; - CellMgrImpl( CellBlkImpl const & cellBlks ) + CellMgrImpl( CellBlkImpl && cellBlks ) : m_cellBlk( cellBlks ) { } - [[nodiscard]] std::list< CellBlk const * > getCellBlks() const override; + [[nodiscard]] std::map< string, generators::CellBlk const * > getCellBlks() const override; private: CellBlkImpl m_cellBlk; @@ -199,30 +342,52 @@ class MeshMappingImpl : public generators::MeshMappings : MeshMappings( name, parent ) { } - [[nodiscard]] generators::CellMgr const & getCellMgr() const override; + [[nodiscard]] generators::CellMgr const & getCellMgr() const override + { + return m_cellMgr; + } + + [[nodiscard]] generators::EdgeMgr const & getEdgeMgr() const override + { + return m_edgeMgr; + } + + [[nodiscard]] generators::FaceMgr const & getFaceMgr() const override + { + return m_faceMgr; + } - [[nodiscard]] generators::EdgeMgr const & getEdgeMgr() const override; + [[nodiscard]] generators::NodeMgr const & getNodeMgr() const override + { + return m_nodeMgr; + } - [[nodiscard]] generators::FaceMgr const & getFaceMgr() const override; + [[nodiscard]] std::set< integer > const & getNeighbors() const override + { + return m_neighbors; + } - [[nodiscard]] generators::NodeMgr const & getNodeMgr() const override; + void setNeighbors( std::set< integer > && neighbors ) + { + m_neighbors = std::move( neighbors ); + } - void setCellMgr( CellMgrImpl const & cellMgr ) + void setCellMgr( CellMgrImpl && cellMgr ) { m_cellMgr = cellMgr; } - void setEdgeMgr( EdgeMgrImpl const & edgeMgr ) + void setEdgeMgr( EdgeMgrImpl && edgeMgr ) { m_edgeMgr = edgeMgr; } - void setFaceMgr( FaceMgrImpl const & faceMgr ) + void setFaceMgr( FaceMgrImpl && faceMgr ) { m_faceMgr = faceMgr; } - void setNodeMgr( NodeMgrImpl const & nodeMgr ) + void setNodeMgr( NodeMgrImpl && nodeMgr ) { m_nodeMgr = nodeMgr; } @@ -232,6 +397,8 @@ class MeshMappingImpl : public generators::MeshMappings EdgeMgrImpl m_edgeMgr; FaceMgrImpl m_faceMgr; NodeMgrImpl m_nodeMgr; + + std::set< integer > m_neighbors; }; } // geos diff --git a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp index bda2033f301..e9e5180d967 100644 --- a/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp +++ b/src/coreComponents/mesh/generators/VTKMeshGenerator.cpp @@ -89,7 +89,6 @@ void VTKMeshGenerator::fillMeshMappings( MeshMappingImpl & meshMappings, GEOS_LOG_RANK_0( GEOS_FMT( "{} '{}': reading mesh from {}", catalogName(), getName(), m_filePath ) ); { - GEOS_LOG_LEVEL_RANK_0( 2, " reading the dataset..." ); vtk::AllMeshes allMeshes = vtk::loadAllMeshes( m_filePath, m_mainBlockName, m_faceBlockNames ); GEOS_LOG_LEVEL_RANK_0( 2, " redistributing mesh..." ); diff --git a/src/coreComponents/mesh/generators/include/CellBlk.hpp b/src/coreComponents/mesh/generators/include/CellBlk.hpp index 57fd2d1d52b..c269f3cc074 100644 --- a/src/coreComponents/mesh/generators/include/CellBlk.hpp +++ b/src/coreComponents/mesh/generators/include/CellBlk.hpp @@ -15,14 +15,16 @@ #ifndef GEOS_CELLBLK_HPP #define GEOS_CELLBLK_HPP +#include "GhostExchange.hpp" + #include "mesh/ElementType.hpp" #include "common/DataTypes.hpp" -namespace geos +namespace geos::generators { -class CellBlk +class CellBlk: public generators::GhostExchange { public: @@ -75,12 +77,6 @@ class CellBlk * @return The mapping relationship as a 2d-array. */ [[nodiscard]] virtual array2d< localIndex > getElemToFaces() const = 0; - - /** - * @brief Get local to global map. - * @return The mapping relationship as an array. - */ - [[nodiscard]] virtual array1d< globalIndex > localToGlobalMap() const = 0; }; } // end of namespace diff --git a/src/coreComponents/mesh/generators/include/CellMgr.hpp b/src/coreComponents/mesh/generators/include/CellMgr.hpp index 69013a6d822..4be492df7bc 100644 --- a/src/coreComponents/mesh/generators/include/CellMgr.hpp +++ b/src/coreComponents/mesh/generators/include/CellMgr.hpp @@ -29,7 +29,7 @@ namespace geos::generators class CellMgr { public: - virtual std::list< CellBlk const * > getCellBlks() const = 0; + virtual std::map< string, CellBlk const * > getCellBlks() const = 0; // // /** // * @brief Returns a group containing the cell blocks as @p CellBlockABC instances. diff --git a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp index fc451e74f7f..dd01c477659 100644 --- a/src/coreComponents/mesh/generators/include/EdgeMgr.hpp +++ b/src/coreComponents/mesh/generators/include/EdgeMgr.hpp @@ -15,12 +15,14 @@ #ifndef GEOS_EDGEMGR_HPP #define GEOS_EDGEMGR_HPP +#include "GhostExchange.hpp" + #include "common/DataTypes.hpp" namespace geos::generators { -class EdgeMgr +class EdgeMgr: public GhostExchange { public: /** @@ -40,15 +42,6 @@ class EdgeMgr * @return A one to many relationship. */ [[nodiscard]] virtual ArrayOfArrays< localIndex > getEdgeToFaces() const = 0; - - /** - * @brief Returns the ghost rank mapping. Index is an edge index local to the MPI rank. - * @return A @c numEdges length array. - */ - [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; - - // TODO Use inheritance? - [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; }; } diff --git a/src/coreComponents/mesh/generators/include/FaceMgr.hpp b/src/coreComponents/mesh/generators/include/FaceMgr.hpp index 7fd0add32c8..af0e64960ae 100644 --- a/src/coreComponents/mesh/generators/include/FaceMgr.hpp +++ b/src/coreComponents/mesh/generators/include/FaceMgr.hpp @@ -17,12 +17,14 @@ #include "../CellBlockUtilities.hpp" // TODO At least part of it should become public. +#include "GhostExchange.hpp" + #include "common/DataTypes.hpp" namespace geos::generators { -class FaceMgr +class FaceMgr: public GhostExchange { public: /** @@ -50,10 +52,6 @@ class FaceMgr * In case the face only belongs to one single element, the second value of the table is -1. */ [[nodiscard]] virtual ToCellRelation< array2d< localIndex > > getFaceToElements() const = 0; - - [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; - - [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; }; } diff --git a/src/coreComponents/mesh/generators/include/GhostExchange.hpp b/src/coreComponents/mesh/generators/include/GhostExchange.hpp new file mode 100644 index 00000000000..e8d1a09cb65 --- /dev/null +++ b/src/coreComponents/mesh/generators/include/GhostExchange.hpp @@ -0,0 +1,59 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2020- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#ifndef GEOS_GHOSTEXCHANGE_HPP +#define GEOS_GHOSTEXCHANGE_HPP + +#include "common/DataTypes.hpp" + +namespace geos::generators +{ + +class GhostExchange +{ +public: + /** + * @brief Get the ghost rank mapping. Index is an edge index local to the MPI rank. + * @return A array matching the size of the contained geometrical quantity (nodes, edges...). + */ + [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; + + /** + * @brief Get local to global map for the contained geometrical quantity (nodes, edges...). + * @return The mapping relationship as an array (local indexing is contiguous). + */ + [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; + + /** + * @brief Get global to local map for the contained geometrical quantity (nodes, edges...). + * @return The mapping relationship as a map (global indexing is not contiguous). + */ + [[nodiscard]] virtual unordered_map< globalIndex, localIndex > getGlobalToLocal() const = 0; + + /** + * @brief Get the list of geometrical quantity (nodes, edges...) that need to be sent to neighbors. + * @return A mapping with the neighbor rank as key and the list of entities as value. + */ + [[nodiscard]] virtual std::map< integer, array1d< localIndex > > getSend() const = 0; + + /** + * @brief Get the list of geometrical quantity (nodes, edges...) that will be received from neighbors. + * @return A mapping with the neighbor rank as key and the list of entities as value. + */ + [[nodiscard]] virtual std::map< integer, array1d< localIndex > > getRecv() const = 0; +}; + +} + +#endif //GEOS_GHOSTEXCHANGE_HPP diff --git a/src/coreComponents/mesh/generators/include/MeshMappings.hpp b/src/coreComponents/mesh/generators/include/MeshMappings.hpp index 6da8d0ebbbd..a10cc235c09 100644 --- a/src/coreComponents/mesh/generators/include/MeshMappings.hpp +++ b/src/coreComponents/mesh/generators/include/MeshMappings.hpp @@ -85,6 +85,8 @@ class MeshMappings : public dataRepository::Group virtual EdgeMgr const & getEdgeMgr() const = 0 ; virtual FaceMgr const & getFaceMgr() const = 0 ; virtual NodeMgr const & getNodeMgr() const = 0 ; + + virtual std::set< integer > const & getNeighbors() const = 0; }; } diff --git a/src/coreComponents/mesh/generators/include/NodeMgr.hpp b/src/coreComponents/mesh/generators/include/NodeMgr.hpp index 0ba69c1383b..3ed59330183 100644 --- a/src/coreComponents/mesh/generators/include/NodeMgr.hpp +++ b/src/coreComponents/mesh/generators/include/NodeMgr.hpp @@ -18,12 +18,14 @@ #include "../CellBlockUtilities.hpp" // TODO At least part of it should become public. +#include "GhostExchange.hpp" + #include "common/DataTypes.hpp" namespace geos::generators { -class NodeMgr +class NodeMgr: public GhostExchange { public: /** @@ -58,12 +60,6 @@ class NodeMgr */ [[nodiscard]] virtual ToCellRelation< ArrayOfArrays< localIndex > > getNodeToElements() const = 0; - /** - * @brief The node to global mapping for nodes. - * @return The mapping as an array of size numNodes. - */ - [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; - /** * @brief Returns the node sets. Key of the map is the name of the set. * @return A reference to constant map. From 87054524922d02ced560b3f938eb8f18096c7c8e Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:15:42 -0700 Subject: [PATCH 092/106] remove elemRegionManager from setGeometricalRelations and use the cell block to sub region mapping directly. --- .../mainInterface/ProblemManager.cpp | 6 +++--- .../mesh/ElementRegionManager.cpp | 3 ++- .../mesh/ElementRegionManager.hpp | 19 ++++++++++++++++--- src/coreComponents/mesh/FaceManager.cpp | 5 ++--- src/coreComponents/mesh/FaceManager.hpp | 4 ++-- src/coreComponents/mesh/NodeManager.cpp | 5 ++--- src/coreComponents/mesh/NodeManager.hpp | 4 ++-- 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/coreComponents/mainInterface/ProblemManager.cpp b/src/coreComponents/mainInterface/ProblemManager.cpp index 8407b05590f..6997ea927d9 100644 --- a/src/coreComponents/mainInterface/ProblemManager.cpp +++ b/src/coreComponents/mainInterface/ProblemManager.cpp @@ -586,10 +586,10 @@ void generateMeshLevelFreeFct( generators::MeshMappings const & meshMappings, FaceManager & faceManager = meshLevel.getFaceManager(); ElementRegionManager & elemRegionManager = meshLevel.getElemManager(); - elemRegionManager.generateMesh( meshMappings ); - nodeManager.setGeometricalRelations( meshMappings, elemRegionManager ); + array2d< localIndex > const cb2sr = elemRegionManager.generateMesh( meshMappings ); + nodeManager.setGeometricalRelations( meshMappings, cb2sr.toViewConst() ); edgeManager.setGeometricalRelations( meshMappings.getNeighbors(), meshMappings.getEdgeMgr() ); - faceManager.setGeometricalRelations( meshMappings, elemRegionManager, nodeManager ); + faceManager.setGeometricalRelations( meshMappings, cb2sr.toViewConst(), nodeManager ); // nodeManager.constructGlobalToLocalMap( cellBlockManager ); // TODO Still need to work on the sets. // // Edge, face and element region managers rely on the sets provided by the node manager. diff --git a/src/coreComponents/mesh/ElementRegionManager.cpp b/src/coreComponents/mesh/ElementRegionManager.cpp index 89c2985b975..980764c8840 100644 --- a/src/coreComponents/mesh/ElementRegionManager.cpp +++ b/src/coreComponents/mesh/ElementRegionManager.cpp @@ -153,13 +153,14 @@ void ElementRegionManager::generateMesh( CellBlockManagerABC const & cellBlockMa } -void ElementRegionManager::generateMesh( generators::MeshMappings const & meshMappings ) +array2d< localIndex > ElementRegionManager::generateMesh( generators::MeshMappings const & meshMappings ) { this->forElementRegions< CellElementRegion >( [&]( CellElementRegion & elemRegion ) { elemRegion.generateMesh( meshMappings.getNeighbors(), meshMappings.getCellMgr().getCellBlks() ); } ); // TODO finish the work for fractures. + return this->getCellBlockToSubRegionMap( meshMappings.getCellMgr().getCellBlks() ); } void ElementRegionManager::generateWells( CellBlockManagerABC const & cellBlockManager, diff --git a/src/coreComponents/mesh/ElementRegionManager.hpp b/src/coreComponents/mesh/ElementRegionManager.hpp index fd30fe27331..1f3d5159c69 100644 --- a/src/coreComponents/mesh/ElementRegionManager.hpp +++ b/src/coreComponents/mesh/ElementRegionManager.hpp @@ -149,8 +149,14 @@ class ElementRegionManager : public ObjectManagerBase */ void generateMesh( CellBlockManagerABC const & cellBlockManager ); - - void generateMesh( generators::MeshMappings const & meshMappings ); + /** + * @brief Fill the regions from the geometries defined in @c meshMappings, and produce a map from cell block indices to element region and subregion indices. + * @param meshMappings Contains the geometry mappings. + * @return A (numBlock x 2) array with each row corresponding to a cell block and containing + * region (first entry) and subregion (second entry) indices, or -1 if block was not used. + * @see getCellBlockToSubRegionMap + */ + array2d< localIndex > generateMesh( generators::MeshMappings const & meshMappings ); /** * @brief Generate the wells. @@ -277,7 +283,6 @@ class ElementRegionManager : public ObjectManagerBase * region (first entry) and subregion (second entry) indices, or -1 if block was not used. */ array2d< localIndex > getCellBlockToSubRegionMap( CellBlockManagerABC const & cellBlockManager ) const; - array2d< localIndex > getCellBlockToSubRegionMap( std::map< string, generators::CellBlk const * > const & cb ) const; /** * @brief This function is used to launch kernel function over all the element regions with region type = @@ -1128,6 +1133,14 @@ class ElementRegionManager : public ObjectManagerBase private: + /** + * @brief Produce a map from cell block indices to element region and subregion indices + * @param[in] cb Maps names to the cell blocks. + * @return a (numBlock x 2) array with each row corresponding to a cell block and containing + * region (first entry) and subregion (second entry) indices, or -1 if block was not used. + */ + array2d< localIndex > getCellBlockToSubRegionMap( std::map< string, generators::CellBlk const * > const & cb ) const; + /** * @brief Pack a list of wrappers or get the buffer size needed to pack. * @param buffer pointer to the buffer to be packed diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index f14139119c7..5339985b762 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -207,7 +207,7 @@ void FaceManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock } void FaceManager::setGeometricalRelations( generators::MeshMappings const & meshMappings, - ElementRegionManager const & elemRegionManager, + arrayView2d< localIndex const > const & cb2sr, NodeManager const & nodeManager ) { GEOS_MARK_FUNCTION; @@ -226,8 +226,7 @@ void FaceManager::setGeometricalRelations( generators::MeshMappings const & mesh copyExchangeInfo( meshMappings.getNeighbors(), faceMgr.getGhostRank(), faceMgr.getSend(), faceMgr.getRecv() ); ToCellRelation< array2d< localIndex > > const toCellBlock = faceMgr.getFaceToElements(); - array2d< localIndex > const blockToSubRegion = elemRegionManager.getCellBlockToSubRegionMap( meshMappings.getCellMgr().getCellBlks() ); // TODO This already exists in NodeManager - meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( blockToSubRegion.toViewConst(), + meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( cb2sr.toViewConst(), toCellBlock, m_toElements ); // TODO add the fracture stuff diff --git a/src/coreComponents/mesh/FaceManager.hpp b/src/coreComponents/mesh/FaceManager.hpp index f0362246062..2d696623b0f 100644 --- a/src/coreComponents/mesh/FaceManager.hpp +++ b/src/coreComponents/mesh/FaceManager.hpp @@ -127,13 +127,13 @@ class FaceManager : public ObjectManagerBase /** * @brief Initialise the current @c FaceManager with the information from @c meshMappings. * @param[in] meshMappings Provides all the geometrical mappings. - * @param[in] elemRegionManager Provides the mapping from the incoming geometrical cell blocks to their sub-region twin. + * @param[in] cb2sr The mapping from the incoming geometrical cell blocks to their sub-region twin. * @param[in] nodeManager Provides the positions of the nodes in order to perform the area computations. * @note We could have relied on the node positions of the @c meshMappings in order to reduce the number of input parameters. * This is disputable since we do not want to have multiple sources of data which, while currenly identical, may in the future differ. */ void setGeometricalRelations( generators::MeshMappings const & meshMappings, - ElementRegionManager const & elemRegionManager, + arrayView2d< localIndex const > const & cb2sr, NodeManager const & nodeManager ); /** diff --git a/src/coreComponents/mesh/NodeManager.cpp b/src/coreComponents/mesh/NodeManager.cpp index 116d26e5301..88f2bb97d98 100644 --- a/src/coreComponents/mesh/NodeManager.cpp +++ b/src/coreComponents/mesh/NodeManager.cpp @@ -194,7 +194,7 @@ void NodeManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock } void NodeManager::setGeometricalRelations( generators::MeshMappings const & meshMappings, - ElementRegionManager const & elemRegionManager ) + arrayView2d< localIndex const > const & cb2sr ) { GEOS_MARK_FUNCTION; auto const & nodeMgr = meshMappings.getNodeMgr(); @@ -216,8 +216,7 @@ void NodeManager::setGeometricalRelations( generators::MeshMappings const & mesh LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); ToCellRelation< ArrayOfArrays< localIndex > > const toCellBlock = nodeMgr.getNodeToElements(); - array2d< localIndex > const blockToSubRegion = elemRegionManager.getCellBlockToSubRegionMap( meshMappings.getCellMgr().getCellBlks() ); // TODO This already exists in FaceManager - meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( blockToSubRegion.toViewConst(), + meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( cb2sr.toViewConst(), toCellBlock, m_toElements ); // TODO add the fracture component diff --git a/src/coreComponents/mesh/NodeManager.hpp b/src/coreComponents/mesh/NodeManager.hpp index c9e9027ec84..3b600ab55a7 100644 --- a/src/coreComponents/mesh/NodeManager.hpp +++ b/src/coreComponents/mesh/NodeManager.hpp @@ -169,10 +169,10 @@ class NodeManager : public ObjectManagerBase /** * @brief Initialise the current @c NodeManager with the information from @c meshMappings. * @param[in] meshMappings Provides all the geometrical mappings. - * @param[in] elemRegionManager Provides the mapping from the incoming geometrical cell blocks to their sub-region twin. + * @param[in] cb2sr The mapping from the incoming geometrical cell blocks to their sub-region twin. */ void setGeometricalRelations( generators::MeshMappings const & meshMappings, - ElementRegionManager const & elemRegionManager ); + arrayView2d< localIndex const > const & cb2sr ); /** * @brief Link the current manager to other managers. From fbe4bb3aabdd49312c7feed411043702ad25005f Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:16:16 -0700 Subject: [PATCH 093/106] Extract the ghostRank from the send/recv instead of duplicating it. --- .../mesh/CellElementSubRegion.cpp | 2 +- src/coreComponents/mesh/EdgeManager.cpp | 2 +- src/coreComponents/mesh/FaceManager.cpp | 2 +- src/coreComponents/mesh/NodeManager.cpp | 2 +- src/coreComponents/mesh/ObjectManagerBase.cpp | 19 +++++- src/coreComponents/mesh/ObjectManagerBase.hpp | 2 - .../mesh/generators/BuildPods.cpp | 65 +------------------ src/coreComponents/mesh/generators/Pods.cpp | 12 ++-- src/coreComponents/mesh/generators/Pods.hpp | 25 ------- .../mesh/generators/include/GhostExchange.hpp | 6 -- 10 files changed, 26 insertions(+), 111 deletions(-) diff --git a/src/coreComponents/mesh/CellElementSubRegion.cpp b/src/coreComponents/mesh/CellElementSubRegion.cpp index 1cb6fb931e9..ecc02abcc50 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.cpp +++ b/src/coreComponents/mesh/CellElementSubRegion.cpp @@ -131,7 +131,7 @@ void CellElementSubRegion::copyFromCellBlock( std::set< integer > const & neighb this->m_localToGlobalMap = cellBlock.getLocalToGlobal(); this->m_globalToLocalMap = cellBlock.getGlobalToLocal(); - copyExchangeInfo( neighbors, cellBlock.getGhostRank(), cellBlock.getSend(), cellBlock.getRecv() ); + copyExchangeInfo( neighbors, cellBlock.getSend(), cellBlock.getRecv() ); //this->constructGlobalToLocalMap(); // TODO what about `m_localMaxGlobalIndex`? // TODO add the external properties import diff --git a/src/coreComponents/mesh/EdgeManager.cpp b/src/coreComponents/mesh/EdgeManager.cpp index 4140104371e..9a2e99dd931 100644 --- a/src/coreComponents/mesh/EdgeManager.cpp +++ b/src/coreComponents/mesh/EdgeManager.cpp @@ -119,7 +119,7 @@ void EdgeManager::setGeometricalRelations( std::set< integer > const & neighbors m_globalToLocalMap = edgeMgr.getGlobalToLocal(); // TODO not for there, but it's convenient - copyExchangeInfo( neighbors, edgeMgr.getGhostRank(), edgeMgr.getSend(), edgeMgr.getRecv() ); + copyExchangeInfo( neighbors, edgeMgr.getSend(), edgeMgr.getRecv() ); } void EdgeManager::setupRelatedObjectsInRelations( NodeManager const & nodeManager, diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index 5339985b762..5da8862ba0c 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -223,7 +223,7 @@ void FaceManager::setGeometricalRelations( generators::MeshMappings const & mesh m_globalToLocalMap = faceMgr.getGlobalToLocal(); // TODO not for there, but it's convenient - copyExchangeInfo( meshMappings.getNeighbors(), faceMgr.getGhostRank(), faceMgr.getSend(), faceMgr.getRecv() ); + copyExchangeInfo( meshMappings.getNeighbors(), faceMgr.getSend(), faceMgr.getRecv() ); ToCellRelation< array2d< localIndex > > const toCellBlock = faceMgr.getFaceToElements(); meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( cb2sr.toViewConst(), diff --git a/src/coreComponents/mesh/NodeManager.cpp b/src/coreComponents/mesh/NodeManager.cpp index 88f2bb97d98..b598cb351d9 100644 --- a/src/coreComponents/mesh/NodeManager.cpp +++ b/src/coreComponents/mesh/NodeManager.cpp @@ -208,7 +208,7 @@ void NodeManager::setGeometricalRelations( generators::MeshMappings const & mesh m_globalToLocalMap = nodeMgr.getGlobalToLocal(); // TODO not for there, but it's convenient - copyExchangeInfo( meshMappings.getNeighbors(), nodeMgr.getGhostRank(), nodeMgr.getSend(), nodeMgr.getRecv() ); + copyExchangeInfo( meshMappings.getNeighbors(), nodeMgr.getSend(), nodeMgr.getRecv() ); m_toEdgesRelation.base().assimilate< parallelHostPolicy >( nodeMgr.getNodeToEdges(), LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); diff --git a/src/coreComponents/mesh/ObjectManagerBase.cpp b/src/coreComponents/mesh/ObjectManagerBase.cpp index 40d56597c3a..845d5cbe1bb 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.cpp +++ b/src/coreComponents/mesh/ObjectManagerBase.cpp @@ -1044,11 +1044,26 @@ void ObjectManagerBase::moveSets( LvArray::MemorySpace const targetSpace ) } void ObjectManagerBase::copyExchangeInfo( std::set< integer > const & neighbors, - array1d< integer > const & ghostRank, std::map< integer, array1d< localIndex > > const & send, std::map< integer, array1d< localIndex > > const & recv ) { - m_ghostRank = ghostRank; + // Build the ghost rank from the `send` and `recv` information. + m_ghostRank.setValues< serialPolicy >( -2 ); + for( auto const & [_, lis]: send ) + { + for( localIndex const & li: lis ) + { + m_ghostRank[li] = -1; + } + } + for( auto const & [rank, lis]: recv ) + { + for( localIndex const & li: lis ) + { + m_ghostRank[li] = rank; + } + } + for( integer const & rank: neighbors ) { addNeighbor( rank ); diff --git a/src/coreComponents/mesh/ObjectManagerBase.hpp b/src/coreComponents/mesh/ObjectManagerBase.hpp index 552c7168495..b41500b5300 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.hpp +++ b/src/coreComponents/mesh/ObjectManagerBase.hpp @@ -927,12 +927,10 @@ class ObjectManagerBase : public dataRepository::Group /** * @brief Copy the ghosting exchange information for the entities (e.g. nodes, edges...) held by the current manager. * @param[in] neighbors The ranks of the neighbors (does not include the current MPI rank) - * @param[in] ghostRank The ghost ranks of the entities own by the current manager. * @param[in] send The entities to send to which neighbors. * @param[in] recv The entities that will be received from which neighbor. */ void copyExchangeInfo( std::set< integer > const & neighbors, - array1d< integer > const & ghostRank, std::map< integer, array1d< localIndex > > const & send, std::map< integer, array1d< localIndex > > const & recv ); /// Group that holds object sets. diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 8edfbcd8b1d..cac2518a799 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -80,26 +80,6 @@ std::tuple< GlobalToLocal, LocalToGlobal > buildL2GMappings( MeshGraph const & g return { g2l, l2g }; } -template< class GI, class LI > -std::vector< integer > buildGhostRank( std::map< GI, LI > const & g2l, - std::map< GI, std::set< MpiRank > > const & send, - std::map< GI, MpiRank > const & recv ) -{ - std::vector< integer > ghostRank( std::size( g2l ), -2 ); - - for( auto const & [gi, _]: send ) - { - ghostRank[g2l.at( gi ).get()] = -1; - } - for( auto const & [gi, rank]: recv ) - { - ghostRank[g2l.at( gi ).get()] = rank.get(); - } - - return ghostRank; -} - - MeshGraph mergeMeshGraph( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts ) @@ -219,7 +199,6 @@ std::map< integer, array1d< localIndex > > toFlavorlessMapping( std::map< MpiRan }; EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, - std::vector< integer > const & ghostRank, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, std::map< EdgeLocIdx, std::vector< FaceLocIdx > > const & e2f, std::map< EdgeGlbIdx, EdgeLocIdx > const & eg2l, @@ -227,7 +206,6 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::map< MpiRank, std::vector< EdgeLocIdx > > const & send, std::map< MpiRank, std::vector< EdgeLocIdx > > const & recv ) { - GEOS_ASSERT_EQ( numEdges, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numEdges, std::size( e2n ) ); GEOS_ASSERT_EQ( numEdges, std::size( e2f ) ); GEOS_ASSERT_EQ( numEdges, std::size( eg2l ) ); @@ -243,15 +221,7 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, auto [l2g, g2l] = convertGlbLoc( eg2l ); - array1d< integer > ghostRank_; - ghostRank_.reserve( numEdges ); - for( integer const & gr: ghostRank ) - { - ghostRank_.emplace_back( gr ); - } - return EdgeMgrImpl( numEdges, - std::move( ghostRank_ ), std::move( e2n_ ), convertToAoA( e2f ), std::move( g2l ), @@ -261,7 +231,6 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, } FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, - std::vector< integer > const & ghostRank, std::map< FaceLocIdx, std::vector< NodeLocIdx > > const & f2n, std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const & f2e, std::map< FaceLocIdx, std::vector< CellLocIdx > > const & f2c, @@ -270,7 +239,6 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, std::map< MpiRank, std::vector< FaceLocIdx > > const & send, std::map< MpiRank, std::vector< FaceLocIdx > > const & recv ) { - GEOS_ASSERT_EQ( numFaces, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numFaces, std::size( f2n ) ); GEOS_ASSERT_EQ( numFaces, std::size( f2e ) ); GEOS_ASSERT_EQ( numFaces, std::size( f2c ) ); @@ -279,14 +247,7 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, auto [l2g, g2l] = convertGlbLoc( fg2l ); - array1d< integer > ghostRank_; - ghostRank_.reserve( numFaces ); - for( integer const & gr: ghostRank ) - { - ghostRank_.emplace_back( gr ); - } - - return FaceMgrImpl( numFaces, std::move( ghostRank_ ), + return FaceMgrImpl( numFaces, convertToAoA( f2n ), convertToAoA( f2e ), convertToA2d( f2c, 2, false ), @@ -297,7 +258,6 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, } NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, - std::vector< integer > const & ghostRank, std::map< NodeGlbIdx, std::array< double, 3 > > const & n2pos, std::map< NodeLocIdx, std::vector< EdgeLocIdx > > const & n2e, std::map< NodeLocIdx, std::vector< FaceLocIdx > > const & n2f, @@ -307,7 +267,6 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, std::map< MpiRank, std::vector< NodeLocIdx > > const & send, std::map< MpiRank, std::vector< NodeLocIdx > > const & recv ) { - GEOS_ASSERT_EQ( numNodes, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numNodes, std::size( n2pos ) ); GEOS_ASSERT_EQ( numNodes, std::size( n2e ) ); GEOS_ASSERT_EQ( numNodes, std::size( n2f ) ); @@ -319,13 +278,6 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock auto [l2g, g2l] = convertGlbLoc( ng2l ); - array1d< integer > ghostRank_; - ghostRank_.reserve( numNodes ); - for( integer const & gr: ghostRank ) - { - ghostRank_.emplace_back( gr ); - } - array2d< real64, nodes::REFERENCE_POSITION_PERM > positions( numNodes, 3 ); for( auto const & [ngi, pos]: n2pos ) { @@ -338,7 +290,6 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, return NodeMgrImpl( intConv< localIndex >( numNodes ), std::move( positions ), - std::move( ghostRank_ ), convertToAoA( n2e ), convertToAoA( n2f ), convertToAoA( n2c ), @@ -349,7 +300,6 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, } CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, - std::vector< integer > const & ghostRank, std::map< CellLocIdx, std::vector< NodeLocIdx > > const & c2n, std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, @@ -358,7 +308,6 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, std::map< MpiRank, std::vector< CellLocIdx > > const & send, std::map< MpiRank, std::vector< CellLocIdx > > const & recv ) { - GEOS_ASSERT_EQ( numCells, std::size( ghostRank ) ); GEOS_ASSERT_EQ( numCells, std::size( c2n ) ); GEOS_ASSERT_EQ( numCells, std::size( c2e ) ); GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); @@ -369,15 +318,7 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock auto [l2g, g2l] = convertGlbLoc( cg2l ); - array1d< integer > ghostRank_; - ghostRank_.reserve( numCells ); - for( integer const & gr: ghostRank ) - { - ghostRank_.emplace_back( gr ); - } - return CellBlkImpl( intConv< localIndex >( numCells ), - std::move( ghostRank_ ), convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, 8 ), convertToA2d( c2e, 12 ), convertToA2d( c2f, 6 ), @@ -613,7 +554,6 @@ void buildPods( MeshGraph const & owned, UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); NodeMgrImpl nodeMgr = makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), - buildGhostRank( g2l.nodes, send.nodes, recv.nodes ), graph.n2pos, upwardMappings.n2e, upwardMappings.n2f, @@ -624,7 +564,6 @@ void buildPods( MeshGraph const & owned, invertGhostRecv( recv.nodes, g2l.nodes ) ); EdgeMgrImpl edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), - buildGhostRank( g2l.edges, send.edges, recv.edges ), downwardMappings.e2n, upwardMappings.e2f, g2l.edges, @@ -633,7 +572,6 @@ void buildPods( MeshGraph const & owned, invertGhostRecv( recv.edges, g2l.edges ) ); FaceMgrImpl faceMgr = makeFlavorlessFaceMgrImpl( std::size( g2l.faces ), - buildGhostRank( g2l.faces, send.faces, recv.faces ), downwardMappings.f2n, downwardMappings.f2e, upwardMappings.f2c, @@ -643,7 +581,6 @@ void buildPods( MeshGraph const & owned, invertGhostRecv( recv.faces, g2l.faces ) ); CellBlkImpl cellBlock = makeFlavorlessCellBlkImpl( std::size( g2l.cells ), - buildGhostRank( g2l.cells, send.cells, recv.cells ), downwardMappings.c2n, downwardMappings.c2e, downwardMappings.c2f, diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index f553a12aeb7..63b6ca83b8b 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -19,7 +19,6 @@ namespace geos NodeMgrImpl::NodeMgrImpl( localIndex numNodes, array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, - array1d< integer > && ghostRank, ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, @@ -27,7 +26,7 @@ NodeMgrImpl::NodeMgrImpl( localIndex numNodes, unordered_map< globalIndex, localIndex > && g2l, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, m_numNodes( numNodes ), m_positions( positions ), m_n2e( n2e ), @@ -52,21 +51,19 @@ ToCellRelation< ArrayOfArrays< localIndex > > NodeMgrImpl::getNodeToElements() c } EdgeMgrImpl::EdgeMgrImpl( std::size_t numEdges, - array1d< integer > && ghostRank, array2d< localIndex > && e2n, ArrayOfArrays< localIndex > && e2f, unordered_map< globalIndex, localIndex > && g2l, array1d< globalIndex > && l2g, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, m_numEdges( numEdges ), m_e2n( e2n ), m_e2f( e2f ) { } FaceMgrImpl::FaceMgrImpl( std::size_t numFaces, - array1d< integer > && ghostRank, ArrayOfArrays< localIndex > && f2n, ArrayOfArrays< localIndex > && f2e, array2d< localIndex > && f2c, @@ -74,7 +71,7 @@ FaceMgrImpl::FaceMgrImpl( std::size_t numFaces, array1d< globalIndex > && l2g, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, m_numFaces( numFaces ), m_f2n( f2n ), m_f2e( f2e ), @@ -98,7 +95,6 @@ ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const } CellBlkImpl::CellBlkImpl( localIndex numCells, - array1d< integer > && ghostRank, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -106,7 +102,7 @@ CellBlkImpl::CellBlkImpl( localIndex numCells, unordered_map< globalIndex, localIndex > && g2l, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( ghostRank ), std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, m_numCells( numCells ), m_c2n( c2n ), m_c2e( c2e ), diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 12759c37023..e8b42a25059 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -30,7 +30,6 @@ namespace geos struct GhostMapping { - array1d< integer > m_ghostRank; array1d< globalIndex > m_l2g; unordered_map< globalIndex, localIndex > m_g2l; std::map< integer, array1d< localIndex > > m_send; @@ -44,7 +43,6 @@ class NodeMgrImpl : public generators::NodeMgr NodeMgrImpl( localIndex numNodes, array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, - array1d< integer > && ghostRank, ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, @@ -81,11 +79,6 @@ class NodeMgrImpl : public generators::NodeMgr } // Diamond - [[nodiscard]] array1d< integer > getGhostRank() const override - { - return m_ghost.m_ghostRank; - } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override { return m_ghost.m_l2g; @@ -121,7 +114,6 @@ class EdgeMgrImpl : public generators::EdgeMgr EdgeMgrImpl() = default; EdgeMgrImpl( std::size_t numEdges, - array1d< integer > && ghostRank, array2d< localIndex > && e2n, ArrayOfArrays< localIndex > && e2f, unordered_map< globalIndex, localIndex > && g2l, @@ -145,11 +137,6 @@ class EdgeMgrImpl : public generators::EdgeMgr } // Diamond - [[nodiscard]] array1d< integer > getGhostRank() const override - { - return m_ghost.m_ghostRank; - } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override { return m_ghost.m_l2g; @@ -183,7 +170,6 @@ class FaceMgrImpl : public generators::FaceMgr FaceMgrImpl() = default; FaceMgrImpl( std::size_t numFaces, - array1d< integer > && ghostRank, ArrayOfArrays< localIndex > && f2n, ArrayOfArrays< localIndex > && f2e, array2d< localIndex > && f2c, @@ -210,11 +196,6 @@ class FaceMgrImpl : public generators::FaceMgr [[nodiscard]] ToCellRelation< array2d< localIndex > > getFaceToElements() const override; // Diamond - [[nodiscard]] array1d< integer > getGhostRank() const override - { - return m_ghost.m_ghostRank; - } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override { return m_ghost.m_l2g; @@ -248,7 +229,6 @@ class CellBlkImpl : public generators::CellBlk CellBlkImpl() = default; CellBlkImpl( localIndex numCells, - array1d< integer > && ghostRank, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -286,11 +266,6 @@ class CellBlkImpl : public generators::CellBlk } // Diamond - [[nodiscard]] array1d< integer > getGhostRank() const override - { - return m_ghost.m_ghostRank; - } - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override { return m_ghost.m_l2g; diff --git a/src/coreComponents/mesh/generators/include/GhostExchange.hpp b/src/coreComponents/mesh/generators/include/GhostExchange.hpp index e8d1a09cb65..b562c6dbab0 100644 --- a/src/coreComponents/mesh/generators/include/GhostExchange.hpp +++ b/src/coreComponents/mesh/generators/include/GhostExchange.hpp @@ -23,12 +23,6 @@ namespace geos::generators class GhostExchange { public: - /** - * @brief Get the ghost rank mapping. Index is an edge index local to the MPI rank. - * @return A array matching the size of the contained geometrical quantity (nodes, edges...). - */ - [[nodiscard]] virtual array1d< integer > getGhostRank() const = 0; - /** * @brief Get local to global map for the contained geometrical quantity (nodes, edges...). * @return The mapping relationship as an array (local indexing is contiguous). From 2a97f46eff95b1ad11604ad2738c0a76ad17da50 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:57:45 -0700 Subject: [PATCH 094/106] Compute the neighbors from the send/recv information. --- src/coreComponents/mainInterface/ProblemManager.cpp | 6 +++--- src/coreComponents/mesh/CellElementRegion.cpp | 5 ++--- src/coreComponents/mesh/CellElementRegion.hpp | 3 +-- src/coreComponents/mesh/CellElementSubRegion.cpp | 5 ++--- src/coreComponents/mesh/CellElementSubRegion.hpp | 3 +-- src/coreComponents/mesh/EdgeManager.cpp | 5 ++--- src/coreComponents/mesh/EdgeManager.hpp | 3 +-- src/coreComponents/mesh/ElementRegionManager.cpp | 2 +- src/coreComponents/mesh/FaceManager.cpp | 6 ++---- src/coreComponents/mesh/FaceManager.hpp | 4 ++-- src/coreComponents/mesh/NodeManager.cpp | 6 ++---- src/coreComponents/mesh/NodeManager.hpp | 4 ++-- src/coreComponents/mesh/ObjectManagerBase.cpp | 10 +++++++--- src/coreComponents/mesh/ObjectManagerBase.hpp | 4 +--- 14 files changed, 29 insertions(+), 37 deletions(-) diff --git a/src/coreComponents/mainInterface/ProblemManager.cpp b/src/coreComponents/mainInterface/ProblemManager.cpp index 6997ea927d9..cf71bee0eff 100644 --- a/src/coreComponents/mainInterface/ProblemManager.cpp +++ b/src/coreComponents/mainInterface/ProblemManager.cpp @@ -587,9 +587,9 @@ void generateMeshLevelFreeFct( generators::MeshMappings const & meshMappings, ElementRegionManager & elemRegionManager = meshLevel.getElemManager(); array2d< localIndex > const cb2sr = elemRegionManager.generateMesh( meshMappings ); - nodeManager.setGeometricalRelations( meshMappings, cb2sr.toViewConst() ); - edgeManager.setGeometricalRelations( meshMappings.getNeighbors(), meshMappings.getEdgeMgr() ); - faceManager.setGeometricalRelations( meshMappings, cb2sr.toViewConst(), nodeManager ); + nodeManager.setGeometricalRelations( meshMappings.getNodeMgr(), cb2sr.toViewConst() ); + edgeManager.setGeometricalRelations( meshMappings.getEdgeMgr() ); + faceManager.setGeometricalRelations( meshMappings.getFaceMgr(), cb2sr.toViewConst(), nodeManager ); // nodeManager.constructGlobalToLocalMap( cellBlockManager ); // TODO Still need to work on the sets. // // Edge, face and element region managers rely on the sets provided by the node manager. diff --git a/src/coreComponents/mesh/CellElementRegion.cpp b/src/coreComponents/mesh/CellElementRegion.cpp index dca59309d23..9ead6a251e3 100644 --- a/src/coreComponents/mesh/CellElementRegion.cpp +++ b/src/coreComponents/mesh/CellElementRegion.cpp @@ -47,8 +47,7 @@ void CellElementRegion::generateMesh( Group const & cellBlocks ) } } -void CellElementRegion::generateMesh( std::set< integer > const & neighbors, - std::map< string, generators::CellBlk const * > const & cellBlks ) +void CellElementRegion::generateMesh( std::map< string, generators::CellBlk const * > const & cellBlks ) { Group & elementSubRegions = this->getGroup( viewKeyStruct::elementSubRegions() ); @@ -56,7 +55,7 @@ void CellElementRegion::generateMesh( std::set< integer > const & neighbors, { CellElementSubRegion & subRegion = elementSubRegions.registerGroup< CellElementSubRegion >( cellBlockName ); generators::CellBlk const * source = cellBlks.at( cellBlockName ); - subRegion.copyFromCellBlock( neighbors, *source ); + subRegion.copyFromCellBlock( *source ); } } diff --git a/src/coreComponents/mesh/CellElementRegion.hpp b/src/coreComponents/mesh/CellElementRegion.hpp index 2aec4f23979..0a3dc6169d0 100644 --- a/src/coreComponents/mesh/CellElementRegion.hpp +++ b/src/coreComponents/mesh/CellElementRegion.hpp @@ -119,10 +119,9 @@ class CellElementRegion : public ElementRegionBase /** * @brief Fill the current @c CellElementRegion with the cell blocks. - * @param neighbors The neighbors of the current @c CellElementRegion * @param cellBlks All the cell blocks. It's the role of the current @c CellElementRegion to pick the one it needs. */ - void generateMesh( std::set< integer > const & neighbors, std::map< string, generators::CellBlk const * > const & cellBlks ); + void generateMesh( std::map< string, generators::CellBlk const * > const & cellBlks ); ///@} diff --git a/src/coreComponents/mesh/CellElementSubRegion.cpp b/src/coreComponents/mesh/CellElementSubRegion.cpp index ecc02abcc50..98abe248d5a 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.cpp +++ b/src/coreComponents/mesh/CellElementSubRegion.cpp @@ -106,8 +106,7 @@ void CellElementSubRegion::copyFromCellBlock( CellBlockABC const & cellBlock ) } ); } -void CellElementSubRegion::copyFromCellBlock( std::set< integer > const & neighbors, - generators::CellBlk const & cellBlock ) +void CellElementSubRegion::copyFromCellBlock( generators::CellBlk const & cellBlock ) { // Defines the (unique) element type of this cell element region, // and its associated number of nodes, edges, faces. @@ -131,7 +130,7 @@ void CellElementSubRegion::copyFromCellBlock( std::set< integer > const & neighb this->m_localToGlobalMap = cellBlock.getLocalToGlobal(); this->m_globalToLocalMap = cellBlock.getGlobalToLocal(); - copyExchangeInfo( neighbors, cellBlock.getSend(), cellBlock.getRecv() ); + copyExchangeInfo( cellBlock.getSend(), cellBlock.getRecv() ); //this->constructGlobalToLocalMap(); // TODO what about `m_localMaxGlobalIndex`? // TODO add the external properties import diff --git a/src/coreComponents/mesh/CellElementSubRegion.hpp b/src/coreComponents/mesh/CellElementSubRegion.hpp index 69914cc61af..5d59aad0097 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.hpp +++ b/src/coreComponents/mesh/CellElementSubRegion.hpp @@ -87,10 +87,9 @@ class CellElementSubRegion : public ElementSubRegionBase /** * @brief Fill the current @c CellElementSubRegion using the @c cellBlock information - * @param[in] neighbors The neighbors' ranks. * @param[in] cellBlock The cell block with all the mappings and ghosting information. */ - void copyFromCellBlock( std::set< integer > const & neighbors, generators::CellBlk const & cellBlock ); + void copyFromCellBlock( generators::CellBlk const & cellBlock ); ///@} diff --git a/src/coreComponents/mesh/EdgeManager.cpp b/src/coreComponents/mesh/EdgeManager.cpp index 9a2e99dd931..1caf9901efa 100644 --- a/src/coreComponents/mesh/EdgeManager.cpp +++ b/src/coreComponents/mesh/EdgeManager.cpp @@ -105,8 +105,7 @@ void EdgeManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); } -void EdgeManager::setGeometricalRelations( std::set< integer > const & neighbors, - generators::EdgeMgr const & edgeMgr ) +void EdgeManager::setGeometricalRelations( generators::EdgeMgr const & edgeMgr ) { GEOS_MARK_FUNCTION; resize( edgeMgr.numEdges() ); @@ -119,7 +118,7 @@ void EdgeManager::setGeometricalRelations( std::set< integer > const & neighbors m_globalToLocalMap = edgeMgr.getGlobalToLocal(); // TODO not for there, but it's convenient - copyExchangeInfo( neighbors, edgeMgr.getSend(), edgeMgr.getRecv() ); + copyExchangeInfo( edgeMgr.getSend(), edgeMgr.getRecv() ); } void EdgeManager::setupRelatedObjectsInRelations( NodeManager const & nodeManager, diff --git a/src/coreComponents/mesh/EdgeManager.hpp b/src/coreComponents/mesh/EdgeManager.hpp index c332f51b544..ca7560b728a 100644 --- a/src/coreComponents/mesh/EdgeManager.hpp +++ b/src/coreComponents/mesh/EdgeManager.hpp @@ -130,10 +130,9 @@ class EdgeManager : public ObjectManagerBase /** * @brief Initialise the current @c EdgeManager with the information from @c edgeMgr. - * @param[in] neighbors The ranks of the neighbors. * @param[in] edgeMgr Provides the geometrical mappings at stake for the @c EdgeManager. */ - void setGeometricalRelations( std::set< integer > const & neighbors, generators::EdgeMgr const & edgeMgr ); + void setGeometricalRelations( generators::EdgeMgr const & edgeMgr ); /** * @brief Link the current manager to other managers. diff --git a/src/coreComponents/mesh/ElementRegionManager.cpp b/src/coreComponents/mesh/ElementRegionManager.cpp index 980764c8840..efaf6d4da90 100644 --- a/src/coreComponents/mesh/ElementRegionManager.cpp +++ b/src/coreComponents/mesh/ElementRegionManager.cpp @@ -157,7 +157,7 @@ array2d< localIndex > ElementRegionManager::generateMesh( generators::MeshMappin { this->forElementRegions< CellElementRegion >( [&]( CellElementRegion & elemRegion ) { - elemRegion.generateMesh( meshMappings.getNeighbors(), meshMappings.getCellMgr().getCellBlks() ); + elemRegion.generateMesh( meshMappings.getCellMgr().getCellBlks() ); } ); // TODO finish the work for fractures. return this->getCellBlockToSubRegionMap( meshMappings.getCellMgr().getCellBlks() ); diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index 5da8862ba0c..d0a16861d18 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -206,13 +206,11 @@ void FaceManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock } } -void FaceManager::setGeometricalRelations( generators::MeshMappings const & meshMappings, +void FaceManager::setGeometricalRelations( generators::FaceMgr const & faceMgr, arrayView2d< localIndex const > const & cb2sr, NodeManager const & nodeManager ) { GEOS_MARK_FUNCTION; - auto const & faceMgr = meshMappings.getFaceMgr(); - resize( faceMgr.numFaces() ); m_toNodesRelation.base() = faceMgr.getFaceToNodes(); @@ -223,7 +221,7 @@ void FaceManager::setGeometricalRelations( generators::MeshMappings const & mesh m_globalToLocalMap = faceMgr.getGlobalToLocal(); // TODO not for there, but it's convenient - copyExchangeInfo( meshMappings.getNeighbors(), faceMgr.getSend(), faceMgr.getRecv() ); + copyExchangeInfo( faceMgr.getSend(), faceMgr.getRecv() ); ToCellRelation< array2d< localIndex > > const toCellBlock = faceMgr.getFaceToElements(); meshMapUtilities::transformCellBlockToRegionMap< parallelHostPolicy >( cb2sr.toViewConst(), diff --git a/src/coreComponents/mesh/FaceManager.hpp b/src/coreComponents/mesh/FaceManager.hpp index 2d696623b0f..4db0d331618 100644 --- a/src/coreComponents/mesh/FaceManager.hpp +++ b/src/coreComponents/mesh/FaceManager.hpp @@ -126,13 +126,13 @@ class FaceManager : public ObjectManagerBase /** * @brief Initialise the current @c FaceManager with the information from @c meshMappings. - * @param[in] meshMappings Provides all the geometrical mappings. + * @param[in] faceMgr Provides all the geometrical mappings and ghost exchange information. * @param[in] cb2sr The mapping from the incoming geometrical cell blocks to their sub-region twin. * @param[in] nodeManager Provides the positions of the nodes in order to perform the area computations. * @note We could have relied on the node positions of the @c meshMappings in order to reduce the number of input parameters. * This is disputable since we do not want to have multiple sources of data which, while currenly identical, may in the future differ. */ - void setGeometricalRelations( generators::MeshMappings const & meshMappings, + void setGeometricalRelations( generators::FaceMgr const & faceMgr, arrayView2d< localIndex const > const & cb2sr, NodeManager const & nodeManager ); diff --git a/src/coreComponents/mesh/NodeManager.cpp b/src/coreComponents/mesh/NodeManager.cpp index b598cb351d9..895c7bdc7be 100644 --- a/src/coreComponents/mesh/NodeManager.cpp +++ b/src/coreComponents/mesh/NodeManager.cpp @@ -193,12 +193,10 @@ void NodeManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock elemRegionManager.forElementRegionsComplete< SurfaceElementRegion >( connectNodesTo2dElements ); } -void NodeManager::setGeometricalRelations( generators::MeshMappings const & meshMappings, +void NodeManager::setGeometricalRelations( generators::NodeMgr const & nodeMgr, arrayView2d< localIndex const > const & cb2sr ) { GEOS_MARK_FUNCTION; - auto const & nodeMgr = meshMappings.getNodeMgr(); - resize( nodeMgr.numNodes() ); m_referencePosition = nodeMgr.getNodePositions(); @@ -208,7 +206,7 @@ void NodeManager::setGeometricalRelations( generators::MeshMappings const & mesh m_globalToLocalMap = nodeMgr.getGlobalToLocal(); // TODO not for there, but it's convenient - copyExchangeInfo( meshMappings.getNeighbors(), nodeMgr.getSend(), nodeMgr.getRecv() ); + copyExchangeInfo( nodeMgr.getSend(), nodeMgr.getRecv() ); m_toEdgesRelation.base().assimilate< parallelHostPolicy >( nodeMgr.getNodeToEdges(), LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); diff --git a/src/coreComponents/mesh/NodeManager.hpp b/src/coreComponents/mesh/NodeManager.hpp index 3b600ab55a7..dd6c2420ec7 100644 --- a/src/coreComponents/mesh/NodeManager.hpp +++ b/src/coreComponents/mesh/NodeManager.hpp @@ -168,10 +168,10 @@ class NodeManager : public ObjectManagerBase /** * @brief Initialise the current @c NodeManager with the information from @c meshMappings. - * @param[in] meshMappings Provides all the geometrical mappings. + * @param[in] nodeMgr Provides all the geometrical mappings and ghost exchange information. * @param[in] cb2sr The mapping from the incoming geometrical cell blocks to their sub-region twin. */ - void setGeometricalRelations( generators::MeshMappings const & meshMappings, + void setGeometricalRelations( generators::NodeMgr const & nodeMgr, arrayView2d< localIndex const > const & cb2sr ); /** diff --git a/src/coreComponents/mesh/ObjectManagerBase.cpp b/src/coreComponents/mesh/ObjectManagerBase.cpp index 845d5cbe1bb..bc38e29eb33 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.cpp +++ b/src/coreComponents/mesh/ObjectManagerBase.cpp @@ -1043,8 +1043,7 @@ void ObjectManagerBase::moveSets( LvArray::MemorySpace const targetSpace ) } ); } -void ObjectManagerBase::copyExchangeInfo( std::set< integer > const & neighbors, - std::map< integer, array1d< localIndex > > const & send, +void ObjectManagerBase::copyExchangeInfo( std::map< integer, array1d< localIndex > > const & send, std::map< integer, array1d< localIndex > > const & recv ) { // Build the ghost rank from the `send` and `recv` information. @@ -1064,7 +1063,12 @@ void ObjectManagerBase::copyExchangeInfo( std::set< integer > const & neighbors, } } - for( integer const & rank: neighbors ) + // Compute the neighbors of the current instance from the `send` and `recv` information. + std::set< integer > neighbors_; + neighbors_.merge( mapKeys< std::set >( send ) ); + neighbors_.merge( mapKeys< std::set >( recv ) ); + + for( integer const & rank: neighbors_ ) { addNeighbor( rank ); } diff --git a/src/coreComponents/mesh/ObjectManagerBase.hpp b/src/coreComponents/mesh/ObjectManagerBase.hpp index b41500b5300..c014cd90eab 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.hpp +++ b/src/coreComponents/mesh/ObjectManagerBase.hpp @@ -926,12 +926,10 @@ class ObjectManagerBase : public dataRepository::Group protected: /** * @brief Copy the ghosting exchange information for the entities (e.g. nodes, edges...) held by the current manager. - * @param[in] neighbors The ranks of the neighbors (does not include the current MPI rank) * @param[in] send The entities to send to which neighbors. * @param[in] recv The entities that will be received from which neighbor. */ - void copyExchangeInfo( std::set< integer > const & neighbors, - std::map< integer, array1d< localIndex > > const & send, + void copyExchangeInfo( std::map< integer, array1d< localIndex > > const & send, std::map< integer, array1d< localIndex > > const & recv ); /// Group that holds object sets. Group m_sets; From 87df3237dcd4febf5f2fab0e9754afc787ac8556 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:20:24 -0700 Subject: [PATCH 095/106] Do not provide the l2g map anymore and let the client compute it from g2l. --- .../mesh/CellElementSubRegion.cpp | 4 +- src/coreComponents/mesh/EdgeManager.cpp | 2 +- src/coreComponents/mesh/FaceManager.cpp | 2 +- src/coreComponents/mesh/NodeManager.cpp | 2 +- src/coreComponents/mesh/ObjectManagerBase.cpp | 8 ++ src/coreComponents/mesh/ObjectManagerBase.hpp | 7 ++ .../mesh/generators/BuildPods.cpp | 107 +++++------------- .../mesh/generators/Indices.hpp | 55 +++++++-- src/coreComponents/mesh/generators/Pods.cpp | 12 +- src/coreComponents/mesh/generators/Pods.hpp | 25 ---- .../mesh/generators/include/GhostExchange.hpp | 6 - 11 files changed, 99 insertions(+), 131 deletions(-) diff --git a/src/coreComponents/mesh/CellElementSubRegion.cpp b/src/coreComponents/mesh/CellElementSubRegion.cpp index 98abe248d5a..c969ccd8788 100644 --- a/src/coreComponents/mesh/CellElementSubRegion.cpp +++ b/src/coreComponents/mesh/CellElementSubRegion.cpp @@ -127,8 +127,8 @@ void CellElementSubRegion::copyFromCellBlock( generators::CellBlk const & cellBl this->edgeList() = cellBlock.getElemToEdges(); this->faceList() = cellBlock.getElemToFaces(); - this->m_localToGlobalMap = cellBlock.getLocalToGlobal(); - this->m_globalToLocalMap = cellBlock.getGlobalToLocal(); + m_globalToLocalMap = cellBlock.getGlobalToLocal(); + this->constructLocalToGlobalMap(); copyExchangeInfo( cellBlock.getSend(), cellBlock.getRecv() ); diff --git a/src/coreComponents/mesh/EdgeManager.cpp b/src/coreComponents/mesh/EdgeManager.cpp index 1caf9901efa..c351cc72136 100644 --- a/src/coreComponents/mesh/EdgeManager.cpp +++ b/src/coreComponents/mesh/EdgeManager.cpp @@ -114,8 +114,8 @@ void EdgeManager::setGeometricalRelations( generators::EdgeMgr const & edgeMgr ) m_toFacesRelation.base().assimilate< parallelHostPolicy >( edgeMgr.getEdgeToFaces(), LvArray::sortedArrayManipulation::UNSORTED_NO_DUPLICATES ); // TODO This is new - m_localToGlobalMap = edgeMgr.getLocalToGlobal(); m_globalToLocalMap = edgeMgr.getGlobalToLocal(); + this->constructLocalToGlobalMap(); // TODO not for there, but it's convenient copyExchangeInfo( edgeMgr.getSend(), edgeMgr.getRecv() ); diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index d0a16861d18..69d61492d1e 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -217,8 +217,8 @@ void FaceManager::setGeometricalRelations( generators::FaceMgr const & faceMgr, m_toEdgesRelation.base() = faceMgr.getFaceToEdges(); // TODO This is new - m_localToGlobalMap = faceMgr.getLocalToGlobal(); m_globalToLocalMap = faceMgr.getGlobalToLocal(); + this->constructLocalToGlobalMap(); // TODO not for there, but it's convenient copyExchangeInfo( faceMgr.getSend(), faceMgr.getRecv() ); diff --git a/src/coreComponents/mesh/NodeManager.cpp b/src/coreComponents/mesh/NodeManager.cpp index 895c7bdc7be..d428c2e9b0d 100644 --- a/src/coreComponents/mesh/NodeManager.cpp +++ b/src/coreComponents/mesh/NodeManager.cpp @@ -202,8 +202,8 @@ void NodeManager::setGeometricalRelations( generators::NodeMgr const & nodeMgr, m_referencePosition = nodeMgr.getNodePositions(); // TODO I add the copy of the g2l and l2g mappings here. This is new - m_localToGlobalMap = nodeMgr.getLocalToGlobal(); m_globalToLocalMap = nodeMgr.getGlobalToLocal(); + this->constructLocalToGlobalMap(); // TODO not for there, but it's convenient copyExchangeInfo( nodeMgr.getSend(), nodeMgr.getRecv() ); diff --git a/src/coreComponents/mesh/ObjectManagerBase.cpp b/src/coreComponents/mesh/ObjectManagerBase.cpp index bc38e29eb33..4754d8abac5 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.cpp +++ b/src/coreComponents/mesh/ObjectManagerBase.cpp @@ -203,6 +203,14 @@ void ObjectManagerBase::constructGlobalToLocalMap() } } +void ObjectManagerBase::constructLocalToGlobalMap() +{ + for( auto const & [gi, li]: m_globalToLocalMap ) + { + m_localToGlobalMap[li] = gi; + } +} + localIndex ObjectManagerBase::packSize( string_array const & wrapperNames, arrayView1d< localIndex const > const & packList, integer const recursive, diff --git a/src/coreComponents/mesh/ObjectManagerBase.hpp b/src/coreComponents/mesh/ObjectManagerBase.hpp index c014cd90eab..27fc0720c1f 100644 --- a/src/coreComponents/mesh/ObjectManagerBase.hpp +++ b/src/coreComponents/mesh/ObjectManagerBase.hpp @@ -931,6 +931,13 @@ class ObjectManagerBase : public dataRepository::Group */ void copyExchangeInfo( std::map< integer, array1d< localIndex > > const & send, std::map< integer, array1d< localIndex > > const & recv ); + + + /** + * @brief Constructs the global to local map from the local to global map. + */ + void constructLocalToGlobalMap(); + /// Group that holds object sets. Group m_sets; diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index cac2518a799..f43fdf4d874 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -34,50 +34,25 @@ struct GlobalToLocal std::map< CellGlbIdx, CellLocIdx > cells; }; -struct LocalToGlobal -{ - std::map< NodeLocIdx, NodeGlbIdx > nodes; - std::map< EdgeLocIdx, EdgeGlbIdx > edges; // TODO Do we want to make this a vector already? It's surely the most efficient way to build it. - std::map< FaceLocIdx, FaceGlbIdx > faces; - std::map< CellLocIdx, CellGlbIdx > cells; -}; - -template< class GI, class LI > -void buildL2GMappings( std::set< GI > const & gis, - std::map< GI, LI > & g2l, - std::map< LI, GI > & l2g ) // TODO we can make it a vector -> simple push_backs will be enough to build it +/** + * @brief Computes a global to local map from the global indices provided in @c gis. + * @tparam GI The global index type. + * @param gis The container of global indices. + * @return The map instance. + */ +template< class GI > +std::map< GI, toLocIdx_t< GI > > buildGlobalToLocalMap( std::set< GI > const & gis ) { - g2l.clear(); - l2g.clear(); + using LI = toLocIdx_t< GI >; + std::map< GI, LI > g2l; LI li{ 0 }; for( GI const & gi: gis ) { - g2l.insert( { gi, li } ); - l2g.insert( { li, gi } ); - ++li; + g2l.insert( { gi, li++ } ); } - // TODO add a check to see if we have the full range of local indices. -} - -std::tuple< GlobalToLocal, LocalToGlobal > buildL2GMappings( MeshGraph const & graph ) -{ - GlobalToLocal g2l; - LocalToGlobal l2g; - - // Use `std::ranges::views::keys` when switching to C++20 - auto const keys = []( auto const & m ) - { - return mapKeys< std::set >( m ); - }; - - buildL2GMappings( keys( graph.n2pos ), g2l.nodes, l2g.nodes ); - buildL2GMappings( keys( graph.e2n ), g2l.edges, l2g.edges ); - buildL2GMappings( keys( graph.f2e ), g2l.faces, l2g.faces ); - buildL2GMappings( keys( graph.c2f ), g2l.cells, l2g.cells ); - - return { g2l, l2g }; + return g2l; } MeshGraph mergeMeshGraph( MeshGraph const & owned, @@ -105,7 +80,6 @@ struct DownwardMappings std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; }; - struct UpwardMappings { std::map< EdgeLocIdx, std::vector< FaceLocIdx > > e2f; @@ -141,7 +115,7 @@ ArrayOfArrays< localIndex > convertToAoA( std::map< T, std::vector< U > > const return t2u_; } -template< class T, class U, class P=camp::make_idx_seq_t< 2 > > +template< class T, class U, class P=typename array2d< localIndex >::Permutation > array2d< localIndex, P > convertToA2d( std::map< T, std::vector< U > > const & t2u, int dimU, bool check = true ) @@ -164,21 +138,16 @@ array2d< localIndex, P > convertToA2d( std::map< T, std::vector< U > > const & t return t2u_; } -template< class GI, class LI > -std::tuple< array1d< globalIndex >, unordered_map< globalIndex, localIndex > > -convertGlbLoc( std::map< GI, LI > const & g2l ) +template< class GI > +unordered_map< globalIndex, localIndex > convertGlobalToLocalMap( std::map< GI, toLocIdx_t< GI > > const & g2l ) { - static_assert( !std::is_same_v< GI, LI > ); - array1d< globalIndex > l2g_( std::size( g2l ) ); - l2g_.setValues< serialPolicy >( -1 ); unordered_map< globalIndex, localIndex > g2l_; for( auto const & [gi, li]: g2l ) { - l2g_[li.get()] = gi.get(); g2l_[gi.get()] = li.get(); } - return { l2g_, g2l_ }; + return g2l_; } template< typename LI > @@ -202,14 +171,12 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, std::map< EdgeLocIdx, std::vector< FaceLocIdx > > const & e2f, std::map< EdgeGlbIdx, EdgeLocIdx > const & eg2l, - std::map< EdgeLocIdx, EdgeGlbIdx > const & el2g, std::map< MpiRank, std::vector< EdgeLocIdx > > const & send, std::map< MpiRank, std::vector< EdgeLocIdx > > const & recv ) { GEOS_ASSERT_EQ( numEdges, std::size( e2n ) ); GEOS_ASSERT_EQ( numEdges, std::size( e2f ) ); GEOS_ASSERT_EQ( numEdges, std::size( eg2l ) ); - GEOS_ASSERT_EQ( numEdges, std::size( el2g ) ); array2d< localIndex > e2n_( numEdges, 2 ); for( int i = 0; i < intConv< int >( numEdges ); ++i ) @@ -219,13 +186,10 @@ EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, e2n_[i][1] = std::get< 1 >( nlis ).get(); } - auto [l2g, g2l] = convertGlbLoc( eg2l ); - return EdgeMgrImpl( numEdges, std::move( e2n_ ), convertToAoA( e2f ), - std::move( g2l ), - std::move( l2g ), + convertGlobalToLocalMap( eg2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); } @@ -235,7 +199,6 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, std::map< FaceLocIdx, std::vector< EdgeLocIdx > > const & f2e, std::map< FaceLocIdx, std::vector< CellLocIdx > > const & f2c, std::map< FaceGlbIdx, FaceLocIdx > const & fg2l, - std::map< FaceLocIdx, FaceGlbIdx > const & fl2g, std::map< MpiRank, std::vector< FaceLocIdx > > const & send, std::map< MpiRank, std::vector< FaceLocIdx > > const & recv ) { @@ -243,16 +206,15 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, GEOS_ASSERT_EQ( numFaces, std::size( f2e ) ); GEOS_ASSERT_EQ( numFaces, std::size( f2c ) ); GEOS_ASSERT_EQ( numFaces, std::size( fg2l ) ); - GEOS_ASSERT_EQ( numFaces, std::size( fl2g ) ); - auto [l2g, g2l] = convertGlbLoc( fg2l ); +// auto [l2g, g2l] = convertGlbLoc( fg2l ); + auto g2l = convertGlobalToLocalMap( fg2l ); return FaceMgrImpl( numFaces, convertToAoA( f2n ), convertToAoA( f2e ), convertToA2d( f2c, 2, false ), - std::move( g2l ), - std::move( l2g ), + convertGlobalToLocalMap( fg2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); } @@ -263,7 +225,6 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, std::map< NodeLocIdx, std::vector< FaceLocIdx > > const & n2f, std::map< NodeLocIdx, std::vector< CellLocIdx > > const & n2c, std::map< NodeGlbIdx, NodeLocIdx > const & ng2l, - std::map< NodeLocIdx, NodeGlbIdx > const & nl2g, std::map< MpiRank, std::vector< NodeLocIdx > > const & send, std::map< MpiRank, std::vector< NodeLocIdx > > const & recv ) { @@ -272,11 +233,6 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, GEOS_ASSERT_EQ( numNodes, std::size( n2f ) ); GEOS_ASSERT_EQ( numNodes, std::size( n2c ) ); GEOS_ASSERT_EQ( numNodes, std::size( ng2l ) ); - GEOS_ASSERT_EQ( numNodes, std::size( nl2g ) ); - - // TODO MISSING cell type. Should be OK, the information is conveyed. - // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock - auto [l2g, g2l] = convertGlbLoc( ng2l ); array2d< real64, nodes::REFERENCE_POSITION_PERM > positions( numNodes, 3 ); for( auto const & [ngi, pos]: n2pos ) @@ -293,8 +249,7 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, convertToAoA( n2e ), convertToAoA( n2f ), convertToAoA( n2c ), - std::move( l2g ), - std::move( g2l ), + convertGlobalToLocalMap( ng2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); } @@ -304,7 +259,6 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, std::map< CellGlbIdx, CellLocIdx > const & cg2l, - std::map< CellLocIdx, CellGlbIdx > const & cl2g, std::map< MpiRank, std::vector< CellLocIdx > > const & send, std::map< MpiRank, std::vector< CellLocIdx > > const & recv ) { @@ -312,18 +266,12 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, GEOS_ASSERT_EQ( numCells, std::size( c2e ) ); GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); - GEOS_ASSERT_EQ( numCells, std::size( cl2g ) ); - - // TODO MISSING cell type. Should be OK, the information is conveyed. - // TODO MISSING get the cell -> numNodesPerElement... from the original CellBlock - auto [l2g, g2l] = convertGlbLoc( cg2l ); return CellBlkImpl( intConv< localIndex >( numCells ), convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, 8 ), convertToA2d( c2e, 12 ), convertToA2d( c2f, 6 ), - std::move( l2g ), - std::move( g2l ), + convertGlobalToLocalMap( cg2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); } @@ -548,7 +496,12 @@ void buildPods( MeshGraph const & owned, { MeshGraph const graph = mergeMeshGraph( owned, present, ghosts ); - auto const [g2l, l2g] = buildL2GMappings( graph ); + GlobalToLocal const g2l{ + buildGlobalToLocalMap( mapKeys< std::set >( graph.n2pos ) ), + buildGlobalToLocalMap( mapKeys< std::set >( graph.e2n ) ), + buildGlobalToLocalMap( mapKeys< std::set >( graph.f2e ) ), + buildGlobalToLocalMap( mapKeys< std::set >( graph.c2f ) ) + }; DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); @@ -559,7 +512,6 @@ void buildPods( MeshGraph const & owned, upwardMappings.n2f, upwardMappings.n2c, g2l.nodes, - l2g.nodes, invertGhostSend( send.nodes, g2l.nodes ), invertGhostRecv( recv.nodes, g2l.nodes ) ); @@ -567,7 +519,6 @@ void buildPods( MeshGraph const & owned, downwardMappings.e2n, upwardMappings.e2f, g2l.edges, - l2g.edges, invertGhostSend( send.edges, g2l.edges ), invertGhostRecv( recv.edges, g2l.edges ) ); @@ -576,7 +527,6 @@ void buildPods( MeshGraph const & owned, downwardMappings.f2e, upwardMappings.f2c, g2l.faces, - l2g.faces, invertGhostSend( send.faces, g2l.faces ), invertGhostRecv( recv.faces, g2l.faces ) ); @@ -585,7 +535,6 @@ void buildPods( MeshGraph const & owned, downwardMappings.c2e, downwardMappings.c2f, g2l.cells, - l2g.cells, invertGhostSend( send.cells, g2l.cells ), invertGhostRecv( recv.cells, g2l.cells ) ); diff --git a/src/coreComponents/mesh/generators/Indices.hpp b/src/coreComponents/mesh/generators/Indices.hpp index 62235a3129e..7634686aa33 100644 --- a/src/coreComponents/mesh/generators/Indices.hpp +++ b/src/coreComponents/mesh/generators/Indices.hpp @@ -37,17 +37,56 @@ OUTPUT intConv( INPUT input ) return LvArray::integerConversion< OUTPUT >( input ); } -using NodeLocIdx = fluent::NamedType< localIndex, struct NodeLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; -using NodeGlbIdx = fluent::NamedType< globalIndex, struct NodeGlbIdxTag, fluent::Comparable, fluent::Printable >; -using EdgeLocIdx = fluent::NamedType< localIndex, struct EdgeLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; -using EdgeGlbIdx = fluent::NamedType< globalIndex, struct EdgeGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; -using FaceLocIdx = fluent::NamedType< localIndex, struct FaceLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; -using FaceGlbIdx = fluent::NamedType< globalIndex, struct FaceGlbIdxTag, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; -using CellLocIdx = fluent::NamedType< localIndex, struct CellLocIdxTag, fluent::Comparable, fluent::Printable, fluent::PreIncrementable >; -using CellGlbIdx = fluent::NamedType< globalIndex, struct CellGlbIdxTag, fluent::Comparable, fluent::Printable >; +using NodeLocIdx = fluent::NamedType< localIndex, struct TagNodeLocIdx, fluent::Comparable, fluent::Printable, fluent::PostIncrementable >; +using NodeGlbIdx = fluent::NamedType< globalIndex, struct TagNodeGlbIdx, fluent::Comparable, fluent::Printable >; +using EdgeLocIdx = fluent::NamedType< localIndex, struct TagEdgeLocIdx, fluent::Comparable, fluent::Printable, fluent::PostIncrementable >; +using EdgeGlbIdx = fluent::NamedType< globalIndex, struct TagEdgeGlbIdx, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; +using FaceLocIdx = fluent::NamedType< localIndex, struct TagFaceLocIdx, fluent::Comparable, fluent::Printable, fluent::PostIncrementable >; +using FaceGlbIdx = fluent::NamedType< globalIndex, struct TagFaceGlbIdx, fluent::Comparable, fluent::Printable, fluent::Addable, fluent::Subtractable, fluent::PreIncrementable >; +using CellLocIdx = fluent::NamedType< localIndex, struct TagCellLocIdx, fluent::Comparable, fluent::Printable, fluent::PostIncrementable >; +using CellGlbIdx = fluent::NamedType< globalIndex, struct TagCellGlbIdx, fluent::Comparable, fluent::Printable >; using MpiRank = fluent::NamedType< int, struct MpiRankTag, fluent::Comparable, fluent::Printable, fluent::Addable >; +namespace // anonymous +{ + +template< typename GI > +struct toLocIdx +{ + static_assert( sizeof( GI ) == 0, "Please specialize your traits class." ); +}; + +template<> +struct toLocIdx< NodeGlbIdx > +{ + using type = NodeLocIdx; +}; + +template<> +struct toLocIdx< EdgeGlbIdx > +{ + using type = EdgeLocIdx; +}; + +template<> +struct toLocIdx< FaceGlbIdx > +{ + using type = FaceLocIdx; +}; + +template<> +struct toLocIdx< CellGlbIdx > +{ + using type = CellLocIdx; +}; + +} // end of namespace + +template< typename GI > +using toLocIdx_t = typename toLocIdx< GI >::type; + + inline NodeLocIdx operator "" _nli( unsigned long long int i ) { return NodeLocIdx{ NodeLocIdx::UnderlyingType( i ) }; diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index 63b6ca83b8b..c9aa3f8ff34 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -22,11 +22,10 @@ NodeMgrImpl::NodeMgrImpl( localIndex numNodes, ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, - array1d< globalIndex > && l2g, unordered_map< globalIndex, localIndex > && g2l, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( g2l ), std::move( send ), std::move( recv ) }, m_numNodes( numNodes ), m_positions( positions ), m_n2e( n2e ), @@ -54,10 +53,9 @@ EdgeMgrImpl::EdgeMgrImpl( std::size_t numEdges, array2d< localIndex > && e2n, ArrayOfArrays< localIndex > && e2f, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( g2l ), std::move( send ), std::move( recv ) }, m_numEdges( numEdges ), m_e2n( e2n ), m_e2f( e2f ) @@ -68,10 +66,9 @@ FaceMgrImpl::FaceMgrImpl( std::size_t numFaces, ArrayOfArrays< localIndex > && f2e, array2d< localIndex > && f2c, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( g2l ), std::move( send ), std::move( recv ) }, m_numFaces( numFaces ), m_f2n( f2n ), m_f2e( f2e ), @@ -98,11 +95,10 @@ CellBlkImpl::CellBlkImpl( localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, - array1d< globalIndex > && l2g, unordered_map< globalIndex, localIndex > && g2l, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) - : m_ghost{ std::move( l2g ), std::move( g2l ), std::move( send ), std::move( recv ) }, + : m_ghost{ std::move( g2l ), std::move( send ), std::move( recv ) }, m_numCells( numCells ), m_c2n( c2n ), m_c2e( c2e ), diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index e8b42a25059..6a22754d5b1 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -30,7 +30,6 @@ namespace geos struct GhostMapping { - array1d< globalIndex > m_l2g; unordered_map< globalIndex, localIndex > m_g2l; std::map< integer, array1d< localIndex > > m_send; std::map< integer, array1d< localIndex > > m_recv; @@ -46,7 +45,6 @@ class NodeMgrImpl : public generators::NodeMgr ArrayOfArrays< localIndex > const & n2e, ArrayOfArrays< localIndex > const & n2f, ArrayOfArrays< localIndex > const & n2c, - array1d< globalIndex > && l2g, unordered_map< globalIndex, localIndex > && g2l, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ); @@ -79,11 +77,6 @@ class NodeMgrImpl : public generators::NodeMgr } // Diamond - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override - { - return m_ghost.m_l2g; - } - [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override { return m_ghost.m_g2l; @@ -117,7 +110,6 @@ class EdgeMgrImpl : public generators::EdgeMgr array2d< localIndex > && e2n, ArrayOfArrays< localIndex > && e2f, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ); @@ -137,11 +129,6 @@ class EdgeMgrImpl : public generators::EdgeMgr } // Diamond - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override - { - return m_ghost.m_l2g; - } - [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override { return m_ghost.m_g2l; @@ -174,7 +161,6 @@ class FaceMgrImpl : public generators::FaceMgr ArrayOfArrays< localIndex > && f2e, array2d< localIndex > && f2c, unordered_map< globalIndex, localIndex > && g2l, - array1d< globalIndex > && l2g, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ); @@ -196,11 +182,6 @@ class FaceMgrImpl : public generators::FaceMgr [[nodiscard]] ToCellRelation< array2d< localIndex > > getFaceToElements() const override; // Diamond - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override - { - return m_ghost.m_l2g; - } - [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override { return m_ghost.m_g2l; @@ -232,7 +213,6 @@ class CellBlkImpl : public generators::CellBlk array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, - array1d< globalIndex > && l2g, unordered_map< globalIndex, localIndex > && m_g2l, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ); @@ -266,11 +246,6 @@ class CellBlkImpl : public generators::CellBlk } // Diamond - [[nodiscard]] array1d< globalIndex > getLocalToGlobal() const override - { - return m_ghost.m_l2g; - } - [[nodiscard]] unordered_map< globalIndex, localIndex > getGlobalToLocal() const override { return m_ghost.m_g2l; diff --git a/src/coreComponents/mesh/generators/include/GhostExchange.hpp b/src/coreComponents/mesh/generators/include/GhostExchange.hpp index b562c6dbab0..cb0c6c44bda 100644 --- a/src/coreComponents/mesh/generators/include/GhostExchange.hpp +++ b/src/coreComponents/mesh/generators/include/GhostExchange.hpp @@ -23,12 +23,6 @@ namespace geos::generators class GhostExchange { public: - /** - * @brief Get local to global map for the contained geometrical quantity (nodes, edges...). - * @return The mapping relationship as an array (local indexing is contiguous). - */ - [[nodiscard]] virtual array1d< globalIndex > getLocalToGlobal() const = 0; - /** * @brief Get global to local map for the contained geometrical quantity (nodes, edges...). * @return The mapping relationship as a map (global indexing is not contiguous). From 4b10843383262c8abc52fe49b4106ff096fd959f Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 21 Jun 2024 18:51:24 -0700 Subject: [PATCH 096/106] Making it work with tpetra and integer matrices. --- scripts/rectangular_trilinos.cpp | 94 +++- .../mesh/generators/NewGhosting.cpp | 449 +++++++++--------- 2 files changed, 321 insertions(+), 222 deletions(-) diff --git a/scripts/rectangular_trilinos.cpp b/scripts/rectangular_trilinos.cpp index 30b14ffdd49..87074045b8e 100644 --- a/scripts/rectangular_trilinos.cpp +++ b/scripts/rectangular_trilinos.cpp @@ -1,5 +1,5 @@ /* - * Compile: mpicxx -std=c++20 -I /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/include/ rectangular_trilinos.cpp -L /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/lib -lepetra -Wl,-rpath /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/lib -o /tmp/rectangular_trilinos + * Compile: mpicxx -std=c++20 -I /tmp/tmp.EtfQgEUXLg/src/cmake-build-main/_deps/namedtype-src/include -I /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/include/ /tmp/tmp.EtfQgEUXLg/scripts/rectangular_trilinos.cpp -L /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/lib -lepetra -lepetraext -ltpetra -lteuchoscomm -lteuchoscore -lkokkoscore -Wl,-rpath /opt/GEOS/GEOS_TPL-263-252-ba785a2/trilinos/lib -o /tmp/rectangular_trilinos * Launch: mpirun -n 3 /tmp/rectangular_trilinos */ @@ -9,12 +9,92 @@ #include #include +#include +#include +#include + +#include +//#include + #include +#include + #include +#include -int main( int argc, - char ** argv ) +//using TriMap = Tpetra::Map< int, long long, Kokkos::DefaultNode::DefaultNodeType >; +//using TriCrsMatrix = Tpetra::CrsMatrix< double, int, long long, Kokkos::DefaultNode::DefaultNodeType >; + +//using TriLocIdx = fluent::NamedType< int, struct TagTriLocIdx, fluent::Arithmetic, fluent::ImplicitlyConvertibleTo >; +//using TriGlbIdx = fluent::NamedType< long long int, struct TagTriGlbIdx, fluent::Arithmetic, fluent::ImplicitlyConvertibleTo >; +//template<> +//struct std::is_signed< TriLocIdx > : std::is_signed< typename TriLocIdx::UnderlyingType > +//{ +//}; + +using TriLocIdx = int; +using TriGlbIdx = long long int; + +using TriMap = Tpetra::Map< TriLocIdx, TriGlbIdx >; +using TriCrsMatrix = Tpetra::CrsMatrix< std::uint32_t, TriLocIdx, TriGlbIdx >; +using TriComm = Teuchos::Comm< int >; + +using Teuchos::RCP; + +template< typename T, typename... ARGS > +Teuchos::RCP< T > make_rcp( ARGS && ... args ) +{ + return Teuchos::rcp( new T( std::forward< ARGS >( args )... ) ); +} + +int mainTpetra( int argc, + char ** argv ) +{ + int rank, size; + MPI_Init( &argc, &argv ); + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + MPI_Comm_size( MPI_COMM_WORLD, &size ); + + { // Scope for MPI race destructors + Tpetra::global_size_t const numGlbCols{ 4 }; + Tpetra::global_size_t const numGlbRows( size ); + + RCP< TriComm const > comm = make_rcp< Teuchos::MpiComm< int > const >( MPI_COMM_WORLD ); + std::vector< TriGlbIdx > const ownedGlbRows{ TriGlbIdx( rank ) }; + + RCP< TriMap const > rowMap = make_rcp< TriMap const >( numGlbRows, + ownedGlbRows.data(), + TriLocIdx( std::size( ownedGlbRows ) ), + TriGlbIdx{ 0 }, + comm ); + RCP< TriCrsMatrix > m = make_rcp< TriCrsMatrix >( rowMap, numGlbCols ); + + std::uint32_t const r = rank; + std::vector< std::uint32_t > const value{ 1 + 10 * r, 2 + 10 * r, 3 + 10 * r, 4 + 10 * r }; + std::vector< TriGlbIdx > bulk{ TriGlbIdx{ 3 }, TriGlbIdx{ 2 }, TriGlbIdx{ 1 }, TriGlbIdx{ 0 } }; + m->insertGlobalValues( TriGlbIdx( rank ), TriLocIdx{ 4 }, value.data(), bulk.data() ); + + RCP< TriMap const > domainMap = make_rcp< TriMap const >( numGlbCols, TriGlbIdx{ 0 }, comm ); + RCP< TriMap const > rangeMap = make_rcp< TriMap const >( numGlbCols, TriGlbIdx{ 0 }, comm ); + + m->fillComplete( domainMap, rangeMap ); + + std::cout << m->getDomainMap() << std::endl; + + std::cout << "glb(rows, cols) = " << m->getGlobalNumRows() << " " << m->getGlobalNumCols() << std::endl; + std::cout << "loc(rows, cols) = " << m->getLocalNumRows() << " " << m->getLocalNumCols() << std::endl; + +// Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/poc-tpetra.mat", m ); + } + + MPI_Finalize(); + + return 0; +} + +int mainEpetra( int argc, + char ** argv ) { int rank, size; MPI_Init( &argc, &argv ); @@ -47,8 +127,16 @@ int main( int argc, std::cout << "loc(rows, cols) = " << m.NumMyRows() << " " << m.NumMyCols() << std::endl; // m.Print( std::cout ); + EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/poc-epetra.mat", m ); MPI_Finalize(); return 0; +} + +int main( int argc, + char ** argv ) +{ + return mainTpetra( argc, argv ); +// return mainEpetra( argc, argv ); } \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index d5a202a164c..45a4f77aee8 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -20,19 +20,16 @@ #include "Pods.hpp" +#include "Indices.hpp" + #include "common/MpiWrapper.hpp" #include "common/DataTypes.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Indices.hpp" +#include +#include +#include +#include +#include #include #include @@ -272,16 +269,29 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > return { std::move( owned ), std::move( present ) }; } -std::unique_ptr< Epetra_CrsMatrix > makeTranspose( Epetra_CrsMatrix & input, - bool makeDataContiguous = true ) +//using TriLocIdx = fluent::NamedType< localIndex, struct TagTriLocIdx, fluent::Arithmetic, fluent::ImplicitlyConvertibleTo >; +//using TriGlbIdx = fluent::NamedType< globalIndex, struct TagTriGlbIdx, fluent::Arithmetic, fluent::ImplicitlyConvertibleTo >; +//template<> +//struct ::std::is_signed< TriLocIdx > : ::std::is_signed< typename TriLocIdx::UnderlyingType > +//{ +//}; + +using TriLocIdx = localIndex; +using TriGlbIdx = globalIndex; + +using TriMap = Tpetra::Map< TriLocIdx, TriGlbIdx >; +// We use matrices of integers for some of our usages +// But it seems that Tpetra wants *signed* integers (at least the way we install it). +// Consider using *unsigned* integers if you manage to make it work/link. +using TriScalarInt = std::int32_t; +using TriCrsMatrix = Tpetra::CrsMatrix< TriScalarInt , TriLocIdx, TriGlbIdx >; +using TriDblCrsMatrix = Tpetra::CrsMatrix< double, TriLocIdx, TriGlbIdx >; +using TriComm = Teuchos::Comm< int >; + +template< typename T, typename... ARGS > +Teuchos::RCP< T > make_rcp( ARGS && ... args ) { - Epetra_RowMatrixTransposer transposer( &input ); // This process does not modify the original `ghosted` matrix. - Epetra_CrsMatrix * tr = nullptr; // The transposer returns a pointer we must handle ourselves. - transposer.CreateTranspose( makeDataContiguous, tr ); - - std::unique_ptr< Epetra_CrsMatrix > ptr; - ptr.reset( tr ); - return ptr; + return Teuchos::rcp( new T( std::forward< ARGS >( args )... ) ); } void to_json( json & j, @@ -303,48 +313,48 @@ void to_json( json & j, } -Epetra_CrsMatrix multiply( int commSize, - Epetra_CrsMatrix const & indicator, - Epetra_CrsMatrix & upward ) +TriCrsMatrix multiply( int commSize, + TriCrsMatrix const & indicator, + TriCrsMatrix const & upward ) { - Epetra_Map const & ownedMap = upward.RowMap(); - Epetra_Map const & mpiMap = indicator.RangeMap(); + Teuchos::RCP< TriMap const > ownedMap = upward.getRowMap(); + Teuchos::RCP< TriMap const > mpiMap = indicator.getRangeMap(); // Upward (n -> e -> f -> c) - Epetra_CrsMatrix result_u0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( upward, false, indicator, true, result_u0_0, false ); - result_u0_0.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-0.mat", result_u0_0 ); + TriCrsMatrix result_u0_0( ownedMap, commSize ); + Tpetra::MatrixMatrix::Multiply( upward, false, indicator, true, result_u0_0, false ); + result_u0_0.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-0.mat", result_u0_0 ); - Epetra_CrsMatrix result_u0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( upward, false, result_u0_0, false, result_u0_1, false ); - result_u0_1.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-1.mat", result_u0_1 ); + TriCrsMatrix result_u0_1( ownedMap, commSize ); + Tpetra::MatrixMatrix::Multiply( upward, false, result_u0_0, false, result_u0_1, false ); + result_u0_1.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-1.mat", result_u0_1 ); - Epetra_CrsMatrix result_u0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( upward, false, result_u0_1, false, result_u0_2, false ); - result_u0_2.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-2.mat", result_u0_2 ); + TriCrsMatrix result_u0_2( ownedMap, commSize ); + Tpetra::MatrixMatrix::Multiply( upward, false, result_u0_1, false, result_u0_2, false ); + result_u0_2.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-2.mat", result_u0_2 ); // Downward (c -> f -> e -> n) - auto tDownward = makeTranspose( upward ); // TODO check the algorithm to understand what's more relevant. - // TODO why do we have to perform the transposition ourselves instead of using the flag from `EpetraExt::MatrixMatrix::Multiply`. - Epetra_CrsMatrix result_d0_0( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_u0_2, false, result_d0_0, false ); - result_d0_0.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-4.mat", result_d0_0 ); + TriCrsMatrix result_d0_0( ownedMap, commSize ); + Tpetra::MatrixMatrix::Multiply( upward, true, result_u0_2, false, result_d0_0, false ); + result_d0_0.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-4.mat", result_d0_0 ); + + TriCrsMatrix result_d0_1( ownedMap, commSize ); + Tpetra::MatrixMatrix::Multiply( upward, true, result_d0_0, false, result_d0_1, false ); + result_d0_1.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-5.mat", result_d0_1 ); - Epetra_CrsMatrix result_d0_1( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_d0_0, false, result_d0_1, false ); - result_d0_1.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-5.mat", result_d0_1 ); + TriCrsMatrix result_d0_2( ownedMap, commSize ); + Tpetra::MatrixMatrix::Multiply( upward, true, result_d0_1, false, result_d0_2, false ); + result_d0_2.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-6.mat", result_d0_2 ); - Epetra_CrsMatrix result_d0_2( Epetra_DataAccess::Copy, ownedMap, commSize, false ); - EpetraExt::MatrixMatrix::Multiply( *tDownward, false, result_d0_1, false, result_d0_2, false ); - result_d0_2.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/result-6.mat", result_d0_2 ); + MpiWrapper::barrier(); return result_d0_2; } @@ -365,17 +375,17 @@ class FindGeometricalType FaceGlbIdx const & maxFaceGlbIdx, CellGlbIdx const & maxCellGlbIdx ) : m_edgeOffset( intConv< int >( maxNodeGlbIdx.get() + 1 ) ), - m_faceOffset( intConv< int >( m_edgeOffset + maxEdgeGlbIdx.get() + 1 ) ), - m_cellOffset( intConv< int >( m_faceOffset + maxFaceGlbIdx.get() + 1 ) ), - m_numEntries( intConv< int >( m_cellOffset + maxCellGlbIdx.get() + 1 ) ) + m_faceOffset( intConv< int >( m_edgeOffset + TriGlbIdx( maxEdgeGlbIdx.get() ) + TriGlbIdx( 1 ) ) ), + m_cellOffset( intConv< int >( m_faceOffset + TriGlbIdx( maxFaceGlbIdx.get() ) + TriGlbIdx( 1 ) ) ), + m_numEntries( intConv< int >( m_cellOffset + TriGlbIdx( maxCellGlbIdx.get() ) + TriGlbIdx( 1 ) ) ) { } - [[nodiscard]] int numEntries() const + [[nodiscard]] TriGlbIdx numEntries() const { return m_numEntries; } - [[nodiscard]] Geom getGeometricalType( int const & index ) const + [[nodiscard]] Geom getGeometricalType( TriGlbIdx const & index ) const { if( index < m_edgeOffset ) { @@ -395,51 +405,51 @@ class FindGeometricalType } } - [[nodiscard]] NodeGlbIdx toNodeGlbIdx( int const & index ) const + [[nodiscard]] NodeGlbIdx toNodeGlbIdx( TriGlbIdx const & index ) const { return NodeGlbIdx{ intConv< NodeGlbIdx::UnderlyingType >( index ) }; } - [[nodiscard]] EdgeGlbIdx toEdgeGlbIdx( int const & index ) const + [[nodiscard]] EdgeGlbIdx toEdgeGlbIdx( TriGlbIdx const & index ) const { return EdgeGlbIdx{ intConv< EdgeGlbIdx::UnderlyingType >( index - m_edgeOffset ) }; } - [[nodiscard]] FaceGlbIdx toFaceGlbIdx( int const & index ) const + [[nodiscard]] FaceGlbIdx toFaceGlbIdx( TriGlbIdx const & index ) const { return FaceGlbIdx{ intConv< FaceGlbIdx::UnderlyingType >( index - m_faceOffset ) }; } - [[nodiscard]] CellGlbIdx toCellGlbIdx( int const & index ) const + [[nodiscard]] CellGlbIdx toCellGlbIdx( TriGlbIdx const & index ) const { return CellGlbIdx{ intConv< CellGlbIdx::UnderlyingType >( index - m_cellOffset ) }; } - [[nodiscard]] int fromNodeGlbIdx( NodeGlbIdx const & ngi ) const + [[nodiscard]] TriGlbIdx fromNodeGlbIdx( NodeGlbIdx const & ngi ) const { - return intConv< int >( ngi.get() ); + return TriGlbIdx( ngi.get() ); } - [[nodiscard]] int fromEdgeGlbIdx( EdgeGlbIdx const & egi ) const + [[nodiscard]] TriGlbIdx fromEdgeGlbIdx( EdgeGlbIdx const & egi ) const { - return intConv< int >( egi.get() + m_edgeOffset ); + return TriGlbIdx( egi.get() ) + m_edgeOffset; } - [[nodiscard]] int fromFaceGlbIdx( FaceGlbIdx const & fgi ) const + [[nodiscard]] TriGlbIdx fromFaceGlbIdx( FaceGlbIdx const & fgi ) const { - return intConv< int >( fgi.get() + m_faceOffset ); + return TriGlbIdx( fgi.get() ) + m_faceOffset; } - [[nodiscard]] int fromCellGlbIdx( CellGlbIdx const & cgi ) const + [[nodiscard]] TriGlbIdx fromCellGlbIdx( CellGlbIdx const & cgi ) const { - return intConv< int >( cgi.get() + m_cellOffset ); + return TriGlbIdx( cgi.get() ) + m_cellOffset; } private: - int const m_edgeOffset; - int const m_faceOffset; - int const m_cellOffset; - int const m_numEntries; + TriGlbIdx const m_edgeOffset; + TriGlbIdx const m_faceOffset; + TriGlbIdx const m_cellOffset; + TriGlbIdx const m_numEntries; }; template< std::uint8_t N > @@ -479,12 +489,12 @@ std::array< std::size_t, N > decode( std::size_t const & basis, struct Adjacency { - std::vector< int > ownedNodesIdcs; // TODO Use some strongly typed ints. - std::vector< int > ownedGlbIdcs; // TODO Use some strongly typed ints. - std::vector< int > otherGlbIdcs; // TODO Use some strongly typed ints. - std::vector< int > numEntriesPerRow; - std::vector< std::vector< int > > indices; - std::vector< std::vector< double > > values; + std::vector< TriGlbIdx > ownedNodesIdcs; + std::vector< TriGlbIdx > ownedGlbIdcs; + std::vector< TriGlbIdx > otherGlbIdcs; + Teuchos::Array< std::size_t > numEntriesPerRow; + std::vector< std::vector< TriGlbIdx > > indices; + std::vector< std::vector< TriScalarInt > > values; }; Adjacency buildAdjacency( MeshGraph const & owned, @@ -497,12 +507,12 @@ Adjacency buildAdjacency( MeshGraph const & owned, std::size_t const numOther = std::size( present.n2pos ) + std::size( present.e2n ) + std::size( present.f2e ); // Aliases - std::vector< int > & ownedNodesIdcs = adjacency.ownedNodesIdcs; - std::vector< int > & ownedGlbIdcs = adjacency.ownedGlbIdcs; - std::vector< int > & otherGlbIdcs = adjacency.otherGlbIdcs; - std::vector< int > & numEntriesPerRow = adjacency.numEntriesPerRow; - std::vector< std::vector< int > > & indices = adjacency.indices; - std::vector< std::vector< double > > & values = adjacency.values; + std::vector< TriGlbIdx > & ownedNodesIdcs = adjacency.ownedNodesIdcs; + std::vector< TriGlbIdx > & ownedGlbIdcs = adjacency.ownedGlbIdcs; + std::vector< TriGlbIdx > & otherGlbIdcs = adjacency.otherGlbIdcs; + Teuchos::Array< std::size_t > & numEntriesPerRow = adjacency.numEntriesPerRow; + std::vector< std::vector< TriGlbIdx > > & indices = adjacency.indices; + std::vector< std::vector< TriScalarInt > > & values = adjacency.values; ownedNodesIdcs.reserve( std::size( owned.n2pos ) ); ownedGlbIdcs.reserve( numOwned ); @@ -531,10 +541,10 @@ Adjacency buildAdjacency( MeshGraph const & owned, // Nodes depend on no other geometrical entity, // so we only have one entry `1` in the diagonal of the matrix, // because we add the identity to the adjacency matrix. - int const i = convert.fromNodeGlbIdx( ngi ); + TriGlbIdx const i = convert.fromNodeGlbIdx( ngi ); ownedNodesIdcs.emplace_back( i ); ownedGlbIdcs.emplace_back( i ); - numEntriesPerRow.emplace_back( 0 + 1 ); // `+1` comes from the diagonal + numEntriesPerRow.push_back( 0 + 1 ); // `+1` comes from the diagonal indices.emplace_back( 1, ownedGlbIdcs.back() ); values.emplace_back( 1, ownedGlbIdcs.back() ); } @@ -549,14 +559,14 @@ Adjacency buildAdjacency( MeshGraph const & owned, ownedGlbIdcs.emplace_back( convert.fromEdgeGlbIdx( egi ) ); - numEntriesPerRow.emplace_back( numNodes + 1 ); // `+1` comes from the diagonal - indices.emplace_back( std::vector< int >{ convert.fromNodeGlbIdx( std::get< 0 >( nodes ) ), - convert.fromNodeGlbIdx( std::get< 1 >( nodes ) ), - ownedGlbIdcs.back() } ); + numEntriesPerRow.push_back( numNodes + 1 ); // `+1` comes from the diagonal + indices.emplace_back( std::vector< TriGlbIdx >{ convert.fromNodeGlbIdx( std::get< 0 >( nodes ) ), + convert.fromNodeGlbIdx( std::get< 1 >( nodes ) ), + ownedGlbIdcs.back() } ); // Note that when storing a value that can be `0`, we always add `1`, // (for edges but also later for faces and cells), // to be sure that there will always be some a noticeable figure where we need one. - values.emplace_back( std::vector< double >{ 0 + 1., 1 + 1., numNodes } ); + values.emplace_back( std::vector< TriScalarInt >{ 0 + 1, 1 + 1, numNodes } ); } for( auto const & [fgi, edges]: owned.f2e ) { @@ -571,18 +581,18 @@ Adjacency buildAdjacency( MeshGraph const & owned, ownedGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); - numEntriesPerRow.emplace_back( numEdges + 1 ); // `+1` comes from the diagonal - std::vector< int > & ind = indices.emplace_back( numEntriesPerRow.back() ); - std::vector< double > & val = values.emplace_back( numEntriesPerRow.back() ); + numEntriesPerRow.push_back( numEdges + 1 ); // `+1` comes from the diagonal + std::vector< TriGlbIdx > & ind = indices.emplace_back( numEntriesPerRow.back() ); + std::vector< TriScalarInt > & val = values.emplace_back( numEntriesPerRow.back() ); for( std::size_t i = 0; i < numEdges; ++i ) { EdgeInfo const & edgeInfo = edges[i]; ind[i] = convert.fromEdgeGlbIdx( edgeInfo.index ); std::size_t const v = 1 + encode< 2 >( numEdges, { edgeInfo.start, i } ); - val[i] = double( v ); + val[i] = intConv< TriScalarInt >( v ); } ind.back() = ownedGlbIdcs.back(); - val.back() = double( numEdges ); + val.back() = intConv< TriScalarInt >( numEdges ); } for( auto const & [cgi, faces]: owned.c2f ) { @@ -594,23 +604,23 @@ Adjacency buildAdjacency( MeshGraph const & owned, ownedGlbIdcs.emplace_back( convert.fromCellGlbIdx( cgi ) ); - numEntriesPerRow.emplace_back( numFaces + 1 ); // `+1` comes from the diagonal - std::vector< int > & ind = indices.emplace_back( numEntriesPerRow.back() ); - std::vector< double > & val = values.emplace_back( numEntriesPerRow.back() ); + numEntriesPerRow.push_back( numFaces + 1 ); // `+1` comes from the diagonal + std::vector< TriGlbIdx > & ind = indices.emplace_back( numEntriesPerRow.back() ); + std::vector< TriScalarInt > & val = values.emplace_back( numEntriesPerRow.back() ); for( std::size_t i = 0; i < numFaces; ++i ) { FaceInfo const & faceInfo = faces[i]; ind[i] = convert.fromFaceGlbIdx( faceInfo.index ); std::size_t const v = 1 + encode< 3 >( numFaces, { faceInfo.isFlipped, faceInfo.start, i } ); - val[i] = double( v ); + val[i] = intConv< TriScalarInt >( v ); } ind.back() = ownedGlbIdcs.back(); - val.back() = double( numFaces ); // TODO This should be Hex and the not the number of faces... + val.back() = intConv< TriScalarInt >( numFaces ); // TODO This should be Hex and the not the number of faces... } std::sort( std::begin( ownedGlbIdcs ), std::end( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, std::size( ownedGlbIdcs ) ); - GEOS_ASSERT_EQ( numOwned, std::size( numEntriesPerRow ) ); + GEOS_ASSERT_EQ( numOwned, intConv< std::size_t >( numEntriesPerRow.size() ) ); GEOS_ASSERT_EQ( numOwned, std::size( indices ) ); GEOS_ASSERT_EQ( numOwned, std::size( values ) ); for( std::size_t i = 0; i < numOwned; ++i ) @@ -626,6 +636,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & MaxGlbIdcs const & gis, MpiRank curRank ) { + using Teuchos::RCP; + FindGeometricalType const convert( gis.nodes, gis.edges, gis.faces, gis.cells ); using Geom = FindGeometricalType::Geom; @@ -636,31 +648,31 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::size_t const numOther = std::size( adjacency.otherGlbIdcs ); // Aliases - std::vector< int > const & ownedNodesIdcs = adjacency.ownedNodesIdcs; - std::vector< int > const & ownedGlbIdcs = adjacency.ownedGlbIdcs; - std::vector< int > const & otherGlbIdcs = adjacency.otherGlbIdcs; - std::vector< int > const & numEntriesPerRow = adjacency.numEntriesPerRow; - std::vector< std::vector< int > > const & indices = adjacency.indices; - std::vector< std::vector< double > > const & values = adjacency.values; + std::vector< TriGlbIdx > const & ownedNodesIdcs = adjacency.ownedNodesIdcs; + std::vector< TriGlbIdx > const & ownedGlbIdcs = adjacency.ownedGlbIdcs; + std::vector< TriGlbIdx > const & otherGlbIdcs = adjacency.otherGlbIdcs; + Teuchos::Array< std::size_t > const & numEntriesPerRow = adjacency.numEntriesPerRow; + std::vector< std::vector< TriGlbIdx > > const & indices = adjacency.indices; + std::vector< std::vector< TriScalarInt > > const & values = adjacency.values; - Epetra_MpiComm const & comm = Epetra_MpiComm( MPI_COMM_GEOSX ); - Epetra_Map const ownedMap( n, numOwned, ownedGlbIdcs.data(), 0, comm ); + RCP< TriComm const > const comm = make_rcp< Teuchos::MpiComm< int > const >( MPI_COMM_GEOSX ); + auto const ownedMap = make_rcp< TriMap const >( Tpetra::global_size_t( n ), ownedGlbIdcs.data(), TriLocIdx( numOwned ), TriGlbIdx{ 0 }, comm ); // The `upward` matrix offers a representation of the graph connections. // The matrix is square. Each size being the number of geometrical entities. - Epetra_CrsMatrix upward( Epetra_DataAccess::Copy, ownedMap, numEntriesPerRow.data(), true ); + TriCrsMatrix upward( ownedMap, numEntriesPerRow() ); for( std::size_t i = 0; i < numOwned; ++i ) { - std::vector< int > const & rowIndices = indices[i]; - std::vector< double > const & rowValues = values[i]; - GEOS_ASSERT_EQ( std::size( rowIndices ), std::size_t( numEntriesPerRow[i] ) ); - GEOS_ASSERT_EQ( std::size( rowValues ), std::size_t( numEntriesPerRow[i] ) ); - upward.InsertGlobalValues( ownedGlbIdcs[i], std::size( rowIndices ), rowValues.data(), rowIndices.data() ); + std::vector< TriGlbIdx > const & rowIndices = indices[i]; + std::vector< TriScalarInt > const & rowValues = values[i]; + GEOS_ASSERT_EQ( std::size( rowIndices ), numEntriesPerRow[i] ); + GEOS_ASSERT_EQ( std::size( rowValues ), numEntriesPerRow[i] ); + upward.insertGlobalValues( ownedGlbIdcs[i], TriLocIdx( std::size( rowIndices ) ), rowValues.data(), rowIndices.data() ); } - upward.FillComplete( ownedMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/adj.mat", upward ); + upward.fillComplete( ownedMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/upward.mat", upward ); int const commSize( MpiWrapper::commSize() ); @@ -670,37 +682,36 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // ie the number of geometrical entities in the mesh. // It contains one for every time the geometrical entity is present on the rank. // By present, we do not meant that the ranks owns it: just that it's already available on the rank. - Epetra_Map const mpiMap( commSize, 0, comm ); // Let the current rank get the appropriate index in this map. - Epetra_CrsMatrix indicator( Epetra_DataAccess::Copy, mpiMap, numOwned + numOther, true ); + auto const mpiMap = make_rcp< TriMap const >( Tpetra::global_size_t( commSize ), TriGlbIdx{ 0 }, comm ); // Let the current rank get the appropriate index in this map. + TriCrsMatrix indicator( mpiMap, numOwned + numOther ); - std::vector< double > const ones( n, 1. ); - indicator.InsertGlobalValues( curRank.get(), numOwned, ones.data(), ownedGlbIdcs.data() ); - indicator.InsertGlobalValues( curRank.get(), numOther, ones.data(), otherGlbIdcs.data() ); - indicator.FillComplete( ownedMap, mpiMap ); + std::vector< TriScalarInt > const ones( std::max( { numOwned, numOther, std::size_t( 1 ) } ), 1 ); + indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numOwned ), ones.data(), ownedGlbIdcs.data() ); + indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numOther ), ones.data(), otherGlbIdcs.data() ); + indicator.fillComplete( ownedMap, mpiMap ); // The `ownership` matrix is a diagonal square matrix. // Each size being the number of geometrical entities. // The value of the diagonal term will be the owning rank. // By means of matrices multiplications, it will be possible to exchange the ownerships information across the ranks. // TODO Could we use an Epetra_Vector as a diagonal matrix? - std::vector< double > myRank( 1, curRank.get() ); - Epetra_CrsMatrix ownership( Epetra_DataAccess::Copy, ownedMap, 1, true ); - for( auto const & i: ownedGlbIdcs ) + std::vector< TriScalarInt > myRank( 1, curRank.get() ); + TriCrsMatrix ownership( ownedMap, 1 ); + for( TriGlbIdx const & i: ownedGlbIdcs ) { - ownership.InsertGlobalValues( i, 1, myRank.data(), &i ); + ownership.insertGlobalValues( i, TriLocIdx{ 1 }, myRank.data(), &i ); } - ownership.FillComplete( ownedMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ownership.mat", ownership ); + ownership.fillComplete( ownedMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ownership.mat", ownership ); if( curRank == 0_mpi ) { - GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.NumGlobalCols() ); - GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.NumGlobalRows() ); - GEOS_LOG_RANK( "ownership.NumGlobalCols() = " << ownership.NumGlobalCols() ); - GEOS_LOG_RANK( "ownership.NumGlobalRows() = " << ownership.NumGlobalRows() ); - GEOS_LOG_RANK( "ownership diag = " << std::boolalpha << ownership.LowerTriangular() and ownership.UpperTriangular() ); + GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.getGlobalNumCols() ); + GEOS_LOG_RANK( "indicator.NumGlobalRows() = " << indicator.getGlobalNumRows() ); + GEOS_LOG_RANK( "ownership.NumGlobalCols() = " << ownership.getGlobalNumCols() ); + GEOS_LOG_RANK( "ownership.NumGlobalRows() = " << ownership.getGlobalNumRows() ); } - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/indicator.mat", indicator ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/indicator.mat", indicator ); // The `ghostingFootprint` matrix is rectangular, // the number of columns being the number of MPI ranks, @@ -711,10 +722,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // for the ghosting to be effective. // // From `ghostingFootprint` we can extract where the current rank has to send any owned graph node. - Epetra_CrsMatrix ghostingFootprint( multiply( commSize, indicator, upward ) ); - ghostingFootprint.PutScalar( 1. ); // This can be done after the `FillComplete`, but not with Tpetra! - ghostingFootprint.FillComplete( mpiMap, ownedMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostingFootprint.mat", ghostingFootprint ); + TriCrsMatrix ghostingFootprint( multiply( commSize, indicator, upward ) ); + ghostingFootprint.resumeFill(); + ghostingFootprint.setAllToScalar( 1. ); + ghostingFootprint.fillComplete( mpiMap, ownedMap ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ghostingFootprint.mat", ghostingFootprint ); // We put `1` everywhere there's a non-zero entry, so we'll be able to compose with the `ownership` matrix. // FIXME TODO WARNING From `ghostingFootprint` extract where I have to send @@ -723,8 +735,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & if( curRank == 0_mpi ) { - GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghostingFootprint.NumGlobalCols() ); - GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghostingFootprint.NumGlobalRows() ); + GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghostingFootprint.getGlobalNumCols() ); + GEOS_LOG_RANK( "ghosted->NumGlobalRows() = " << ghostingFootprint.getGlobalNumRows() ); } // The `ghostExchange` matrix is rectangular, @@ -736,36 +748,37 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // the value of the `ghostExchange` matrix term will provide the actual owning rank for all the . // // From `ghostExchange` we can extract which other rank will send to the current rank any graph node. - Epetra_CrsMatrix ghostExchange( Epetra_DataAccess::Copy, mpiMap, 1, false ); - EpetraExt::MatrixMatrix::Multiply( ghostingFootprint, true, ownership, false, ghostExchange, false ); - ghostExchange.FillComplete( ownedMap, mpiMap ); + TriCrsMatrix ghostExchange( mpiMap, 10000 ); // TODO having `0` there should be working! + Tpetra::MatrixMatrix::Multiply( ghostingFootprint, true, ownership, false, ghostExchange, false ); + ghostExchange.fillComplete( ownedMap, mpiMap ); + // TODO Do I have to work with `ghostingFootprint` if I already have `ghostExchange` which may convey more information? // TODO Maybe because of the ownership of the ranks: one is also the "scaled" transposed of the other. if( curRank == 0_mpi ) { - GEOS_LOG_RANK( "ghostExchange->NumGlobalCols() = " << ghostExchange.NumGlobalCols() ); - GEOS_LOG_RANK( "ghostExchange->NumGlobalRows() = " << ghostExchange.NumGlobalRows() ); + GEOS_LOG_RANK( "ghostExchange->NumGlobalCols() = " << ghostExchange.getGlobalNumCols() ); + GEOS_LOG_RANK( "ghostExchange->NumGlobalRows() = " << ghostExchange.getGlobalNumRows() ); } - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/ghostInfo.mat", ghostingFootprint ); + Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ghostInfo.mat", ghostExchange ); - int extracted = 0; - std::vector< double > extractedValues; - std::vector< int > extractedIndices; + std::size_t extracted = 0; + Teuchos::Array< TriScalarInt > extractedValues; + Teuchos::Array< TriGlbIdx > extractedIndices; GhostSend send; - for( int const & index: ownedGlbIdcs ) + for( TriGlbIdx const & index: ownedGlbIdcs ) { - int const length = ghostingFootprint.NumGlobalEntries( index ); + std::size_t const length = ghostingFootprint.getNumEntriesInGlobalRow( index ); extractedValues.resize( length ); extractedIndices.resize( length ); - ghostingFootprint.ExtractGlobalRowCopy( index, length, extracted, extractedValues.data(), extractedIndices.data() ); + ghostingFootprint.getGlobalRowCopy( index, extractedIndices(), extractedValues(), extracted ); GEOS_ASSERT_EQ( extracted, length ); std::set< MpiRank > neighbors; - for( int i = 0; i < extracted; ++i ) + for( std::size_t i = 0; i < extracted; ++i ) { - MpiRank const rank{ extractedIndices[i] }; + MpiRank const rank( extractedIndices[i] ); if( rank != curRank ) { neighbors.insert( rank ); @@ -806,39 +819,39 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } } - int const numNeededIndices = ghostExchange.NumGlobalEntries( curRank.get() ); + std::size_t const numNeededIndices = ghostExchange.getNumEntriesInGlobalRow( TriGlbIdx( curRank.get() ) ); extractedValues.resize( numNeededIndices ); extractedIndices.resize( numNeededIndices ); - ghostExchange.ExtractGlobalRowCopy( curRank.get(), numNeededIndices, extracted, extractedValues.data(), extractedIndices.data() ); + ghostExchange.getGlobalRowCopy( TriGlbIdx( curRank.get() ), extractedIndices(), extractedValues(), extracted ); GEOS_ASSERT_EQ( extracted, numNeededIndices ); - std::set< int > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); - std::set< int > receivedIndices; // The graph nodes that my neighbors will send me. + std::set< TriGlbIdx > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); + std::set< TriGlbIdx > receivedIndices; // The graph nodes that my neighbors will send me. std::set_difference( std::cbegin( allNeededIndices ), std::cend( allNeededIndices ), std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ), std::inserter( receivedIndices, std::end( receivedIndices ) ) ); - std::vector< int > notPresentIndices; // The graphs nodes that are nor owned neither present by/on the current rank. + std::vector< TriGlbIdx > notPresentIndices; // The graphs nodes that are nor owned neither present by/on the current rank. std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), std::back_inserter( notPresentIndices ) ); - std::vector< int > notPresentNodes; + std::vector< TriGlbIdx > notPresentNodes; std::copy_if( std::cbegin( notPresentIndices ), std::cend( notPresentIndices ), - std::back_inserter( notPresentNodes ), [&]( int const & i ) + std::back_inserter( notPresentNodes ), [&]( TriGlbIdx const & i ) { return convert.getGeometricalType( i ) == Geom::NODE; } ); - GEOS_ASSERT_EQ( intConv< int >( std::size( allNeededIndices ) ), numNeededIndices ); + GEOS_ASSERT_EQ( std::size( allNeededIndices ), numNeededIndices ); GhostRecv recv; - for( int i = 0; i < extracted; ++i ) + for( std::size_t i = 0; i < extracted; ++i ) { - int const & index = extractedIndices[i]; + TriGlbIdx const & index = extractedIndices[i]; if( receivedIndices.find( index ) == std::cend( receivedIndices ) ) // TODO make a map `receivedIndices -> mpi rank` { continue; } - MpiRank const sender{ int( extractedValues[i] ) }; + MpiRank const sender( extractedValues[i] ); switch( convert.getGeometricalType( index ) ) { case Geom::NODE: @@ -890,7 +903,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // // Combining `missingIndicator` with the adjacency matrix which conveys a lot of connections information, // we'll be able to create the final `missingIndices` matrix, - // with `range` and `domain` maps (MPI ranks ownerships and offsets) appropriately defined + // with `range` and `domain` maps (MPI ranks ownerships and offsets) are appropriately defined // such that the rows will be available to any ranks that need them (nothing more, nothing less). // // To get the `missingNodePos` matrix which will convey the ghosted nodes positions that are missing on the rank, @@ -912,71 +925,72 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & MpiWrapper::allReduce( numLocMissingCompound, numGlbMissingCompound, 2, MPI_SUM ); std::size_t const & numLocMissingIndices = numLocMissingCompound[0]; std::size_t const & numLocMissingNodes = numLocMissingCompound[1]; - std::size_t const & numGlbMissingIndices = numGlbMissingCompound[0]; - std::size_t const & numGlbMissingNodes = numGlbMissingCompound[1]; + Tpetra::global_size_t const numGlbMissingIndices = numGlbMissingCompound[0]; + Tpetra::global_size_t const numGlbMissingNodes = numGlbMissingCompound[1]; - std::size_t const numGlbNodes = gis.nodes.get() + 1; - std::size_t const numOwnedNodes = std::size( ownedNodesIdcs ); + std::size_t const numGlbNodes = gis.nodes.get() + 1; // TODO Use strongly typed integers + std::size_t const numOwnedNodes = std::size( ownedNodesIdcs ); // TODO Use strongly typed integers - Epetra_Map const missingIndicesMap( intConv< int >( numGlbMissingIndices ), intConv< int >( numLocMissingIndices ), 0, comm ); - Epetra_Map const missingNodesMap( intConv< int >( numGlbMissingNodes ), intConv< int >( numLocMissingNodes ), 0, comm ); + auto const missingIndicesMap = make_rcp< TriMap const >( numGlbMissingIndices, numGlbMissingIndices, TriGlbIdx{ 0 }, comm ); + auto const missingNodesMap = make_rcp< TriMap const >( numGlbMissingNodes, numLocMissingNodes, TriGlbIdx{ 0 }, comm ); // Following information is needed to compute the global row. - int const missingIndicesOffset = *missingIndicesMap.MyGlobalElements(); - int const missingNodesOffset = *missingNodesMap.MyGlobalElements(); + TriGlbIdx const missingIndicesOffset = missingIndicesMap->getGlobalElement( TriLocIdx{ 0 } ); // TODO Hocus Pocus + TriGlbIdx const missingNodesOffset = missingNodesMap->getGlobalElement( TriLocIdx{ 0 } ); // TODO Hocus Pocus // Indicator matrix for the all the missing quantities (nodes, edges, faces and cells). - Epetra_CrsMatrix missingIndicator( Epetra_DataAccess::Copy, missingIndicesMap, 1, true ); + TriCrsMatrix missingIndicator( missingIndicesMap, 1 ); for( std::size_t i = 0; i < numLocMissingIndices; ++i ) { - missingIndicator.InsertGlobalValues( missingIndicesOffset + i, 1, ones.data(), ¬PresentIndices[i] ); + missingIndicator.insertGlobalValues( missingIndicesOffset + TriGlbIdx( i ), TriLocIdx( 1 ), ones.data(), ¬PresentIndices[i] ); } - missingIndicator.FillComplete( ownedMap, missingIndicesMap ); + missingIndicator.fillComplete( ownedMap, missingIndicesMap ); + + // `missingIndices` will contain the missing connectivity information. + TriCrsMatrix missingIndices( missingIndicesMap, 1000 ); // TODO having `0` there should be working! + Tpetra::MatrixMatrix::Multiply( missingIndicator, false, upward, false, missingIndices, false ); + missingIndices.fillComplete( ownedMap, missingIndicesMap ); // Indicator matrix only for the nodes. - Epetra_CrsMatrix missingNodesIndicator( Epetra_DataAccess::Copy, missingNodesMap, 1, true ); - for( int i = 0; i < intConv< int >( numLocMissingNodes ); ++i ) + // Note that it should be an integer matrix full of ones, + // but it's a double matrix because it's going to be multiplied a double matrix. + TriDblCrsMatrix missingNodesIndicator( missingNodesMap, 1 ); + for( std::size_t i = 0; i < numLocMissingNodes; ++i ) { - missingNodesIndicator.InsertGlobalValues( missingNodesOffset + i, 1, ones.data(), ¬PresentNodes[i] ); + double const o = 1.; + missingNodesIndicator.insertGlobalValues( missingNodesOffset + TriGlbIdx( i ), TriLocIdx( 1 ), &o, ¬PresentNodes[i] ); } - Epetra_Map const ownedNodesMap( numGlbNodes, numOwnedNodes, ownedNodesIdcs.data(), 0, comm ); - missingNodesIndicator.FillComplete( ownedNodesMap, missingNodesMap ); + auto const ownedNodesMap = make_rcp< TriMap const >( Tpetra::global_size_t( numGlbNodes ), ownedNodesIdcs.data(), TriLocIdx( numOwnedNodes ), TriGlbIdx{ 0 }, comm ); + missingNodesIndicator.fillComplete( ownedNodesMap, missingNodesMap ); // The `nodePositions` matrix is rectangular. // - Its number of rows is the total number of nodes in the mesh. // - Its number of columns is 3: the x, y, and z coordinates of the nodes. - Epetra_CrsMatrix nodePositions( Epetra_DataAccess::Copy, ownedNodesMap, 3, true ); - std::vector< int > const zot{ 0, 1, 2 }; // zot: zero, one, two. + TriDblCrsMatrix nodePositions( ownedNodesMap, 3 ); + std::vector< TriGlbIdx > const zot{ TriGlbIdx( 0 ), TriGlbIdx( 1 ), TriGlbIdx( 2 ) }; // zot: zero, one, two. for( auto const & [ngi, pos]: owned.n2pos ) { - nodePositions.InsertGlobalValues( convert.fromNodeGlbIdx( ngi ), 3, pos.data(), zot.data() ); + nodePositions.insertGlobalValues( convert.fromNodeGlbIdx( ngi ), TriLocIdx( 3 ), pos.data(), zot.data() ); } - Epetra_Map const threeMap = Epetra_Map( 3, 0, comm ); - nodePositions.FillComplete( threeMap, ownedNodesMap ); - - // `missingIndices` will contain the missing connectivity information. - auto tDownward = makeTranspose( upward ); // TODO give it to multiply! - Epetra_CrsMatrix missingIndices( Epetra_DataAccess::Copy, missingIndicesMap, 1, false ); - EpetraExt::MatrixMatrix::Multiply( missingIndicator, false, *tDownward, true, missingIndices, false ); - missingIndices.FillComplete( ownedMap, missingIndicesMap ); - EpetraExt::RowMatrixToMatrixMarketFile( "/tmp/matrices/missingMappings.mat", missingIndices ); + auto const threeMap = make_rcp< TriMap const >( Tpetra::global_size_t( 3 ), TriGlbIdx( 0 ), comm ); + nodePositions.fillComplete( threeMap, ownedNodesMap ); // `missingNodePos` will contain the missing node positions. - Epetra_CrsMatrix missingNodePos( Epetra_DataAccess::Copy, missingNodesMap, 1, false ); - EpetraExt::MatrixMatrix::Multiply( missingNodesIndicator, false, nodePositions, false, missingNodePos, false ); - missingNodePos.FillComplete( threeMap, missingNodesMap ); + TriDblCrsMatrix missingNodePos( missingNodesMap, 1000 ); // TODO having `0` there should be working! + Tpetra::MatrixMatrix::Multiply( missingNodesIndicator, false, nodePositions, false, missingNodePos, false ); + missingNodePos.fillComplete( threeMap, missingNodesMap ); MeshGraph ghosts; - - for( int i = 0; i < int( numLocMissingIndices ); ++i ) + Teuchos::Array< double > extractedNodePos( 3 ); + for( std::size_t i = 0; i < numLocMissingIndices; ++i ) { - int const index = notPresentIndices[i]; + TriGlbIdx const index = notPresentIndices[i]; Geom const geometricalType = convert.getGeometricalType( index ); - int const length = missingIndices.NumGlobalEntries( missingIndicesOffset + i ); + std::size_t const length = missingIndices.getNumEntriesInGlobalRow( missingIndicesOffset + TriGlbIdx( i ) ); extractedValues.resize( length ); extractedIndices.resize( length ); - missingIndices.ExtractGlobalRowCopy( missingIndicesOffset + i, length, extracted, extractedValues.data(), extractedIndices.data() ); + missingIndices.getGlobalRowCopy( missingIndicesOffset + TriGlbIdx( i ), extractedIndices(), extractedValues(), extracted ); GEOS_ASSERT_EQ( extracted, length ); if( geometricalType == Geom::NODE ) { @@ -985,35 +999,33 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // but we need to extract the position of the node instead. // In order to extract these positions, we use the other matrix `missingNodePos`. GEOS_ASSERT_EQ( length, 1 ); - int const lengthPos = missingNodePos.NumGlobalEntries( missingNodesOffset + i ); - extractedValues.resize( lengthPos ); + std::size_t const lengthPos = missingNodePos.getNumEntriesInGlobalRow( missingNodesOffset + TriGlbIdx( i ) ); + GEOS_ASSERT_EQ( lengthPos, 3 ); extractedIndices.resize( lengthPos ); - missingNodePos.ExtractGlobalRowCopy( missingNodesOffset + i, lengthPos, extracted, extractedValues.data(), extractedIndices.data() ); + missingNodePos.getGlobalRowCopy( missingNodesOffset + TriGlbIdx( i ), extractedIndices(), extractedNodePos(), extracted ); GEOS_ASSERT_EQ( extracted, lengthPos ); - GEOS_ASSERT_EQ( lengthPos, 3 ); - GEOS_ASSERT_EQ( index, notPresentNodes[i] ); std::array< double, 3 > & pos = ghosts.n2pos[convert.toNodeGlbIdx( index )]; for( auto dim = 0; dim < 3; ++dim ) { - pos[extractedIndices[dim]] = extractedValues[dim]; + pos[extractedIndices[dim]] = extractedNodePos[dim]; } continue; } auto const cit = std::find( std::cbegin( extractedIndices ), std::cend( extractedIndices ), index ); - std::ptrdiff_t const numGeomQuantitiesIdx = std::distance( std::cbegin( extractedIndices ), cit ); - int const numGeomQuantities = int( extractedValues[numGeomQuantitiesIdx] ); - GEOS_ASSERT_EQ( extracted, numGeomQuantities + 1 ); + std::size_t const numGeomQuantitiesIdx = intConv< std::size_t >( std::distance( std::cbegin( extractedIndices ), cit ) ); + TriScalarInt const & numGeomQuantities = extractedValues[numGeomQuantitiesIdx]; + GEOS_ASSERT_EQ( extracted, intConv< std::size_t >( numGeomQuantities + 1 ) ); switch( geometricalType ) { case Geom::EDGE: { - int const & numNodes = numGeomQuantities; // Alias + TriScalarInt const & numNodes = numGeomQuantities; // Alias GEOS_ASSERT_EQ( numNodes, 2 ); std::array< NodeGlbIdx, 2 > order{}; - for( int ii = 0; ii < extracted; ++ii ) + for( std::size_t ii = 0; ii < extracted; ++ii ) { if( ii == numGeomQuantitiesIdx ) { @@ -1034,9 +1046,9 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } case Geom::FACE: { - int const & numEdges = numGeomQuantities; // Alias + TriScalarInt const & numEdges = numGeomQuantities; // Alias std::map< integer, EdgeInfo > order; - for( int ii = 0; ii < extracted; ++ii ) + for( std::size_t ii = 0; ii < extracted; ++ii ) { if( ii == numGeomQuantitiesIdx ) { @@ -1061,9 +1073,9 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } case Geom::CELL: { - int const & numFaces = numGeomQuantities; // Alias // TODO This should receive the cell type instead. + TriScalarInt const & numFaces = numGeomQuantities; // Alias // TODO This should receive the cell type instead. std::map< integer, FaceInfo > order; - for( int ii = 0; ii < extracted; ++ii ) + for( std::size_t ii = 0; ii < extracted; ++ii ) { if( ii == numGeomQuantitiesIdx ) { @@ -1082,7 +1094,6 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & { tmp[ord] = faceInfo; } -// GEOS_LOG_RANK( "cells ; index, extIndices, extValues, order = " << index << " | " << json( extIndices ) << " | " << json( extValues ) << " | " << json( order ) ); break; } default: From 77eef650ed324586087d46c9323ab266e25464a4 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:01:56 -0700 Subject: [PATCH 097/106] extractedValues is now a collection of integers, no more float --- src/coreComponents/mesh/generators/NewGhosting.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 45a4f77aee8..06812209a78 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -1033,7 +1033,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } NodeGlbIdx const ngi = convert.toNodeGlbIdx( extractedIndices[ii] ); - integer const ord = integer( extractedValues[ii] - 1 ); + TriScalarInt const ord = extractedValues[ii] - 1; GEOS_ASSERT( ord == 0 or ord == 1 ); order[ord] = ngi; } @@ -1056,7 +1056,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } EdgeGlbIdx const egi = convert.toEdgeGlbIdx( extractedIndices[ii] ); - std::array< std::size_t, 2 > const decoded = decode< 2 >( numEdges, std::size_t( extractedValues[ii] - 1 ) ); + std::array< std::size_t, 2 > const decoded = decode< 2 >( numEdges, extractedValues[ii] - 1 ); order[decoded[1]] = { egi, intConv< std::uint8_t >( decoded[0] ) }; GEOS_ASSERT( decoded[0] == 0 or decoded[0] == 1 ); } @@ -1083,7 +1083,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } FaceGlbIdx const fgi = convert.toFaceGlbIdx( extractedIndices[ii] ); - std::array< std::size_t, 3 > const decoded = decode< 3 >( numFaces, std::size_t( extractedValues[ii] - 1 ) ); + std::array< std::size_t, 3 > const decoded = decode< 3 >( numFaces, extractedValues[ii] - 1 ); order[decoded[2]] = { fgi, intConv< bool >( decoded[0] ), intConv< std::uint8_t >( decoded[1] ) }; } GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numFaces ) ); From bf89fce99302be520ee21173c54e9262268a2a81 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Tue, 25 Jun 2024 14:48:25 -0700 Subject: [PATCH 098/106] Fix sorting inconsistency --- src/coreComponents/mesh/generators/NewGhosting.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 06812209a78..6445889e15c 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -617,7 +617,6 @@ Adjacency buildAdjacency( MeshGraph const & owned, ind.back() = ownedGlbIdcs.back(); val.back() = intConv< TriScalarInt >( numFaces ); // TODO This should be Hex and the not the number of faces... } - std::sort( std::begin( ownedGlbIdcs ), std::end( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, std::size( ownedGlbIdcs ) ); GEOS_ASSERT_EQ( numOwned, intConv< std::size_t >( numEntriesPerRow.size() ) ); @@ -827,9 +826,12 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::set< TriGlbIdx > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); std::set< TriGlbIdx > receivedIndices; // The graph nodes that my neighbors will send me. - std::set_difference( std::cbegin( allNeededIndices ), std::cend( allNeededIndices ), - std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ), - std::inserter( receivedIndices, std::end( receivedIndices ) ) ); + { + std::set< TriGlbIdx > tmp( std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ) ); + std::set_difference( std::cbegin( allNeededIndices ), std::cend( allNeededIndices ), + std::cbegin( tmp ), std::cend( tmp ), + std::inserter( receivedIndices, std::end( receivedIndices ) ) ); + } std::vector< TriGlbIdx > notPresentIndices; // The graphs nodes that are nor owned neither present by/on the current rank. std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), From d5440b9ab8b285845c61fd38467742e576f63ddf Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:04:41 -0700 Subject: [PATCH 099/106] forgot a `const` --- src/coreComponents/mesh/generators/NewGhosting.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 6445889e15c..1ea1af1cc31 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -827,7 +827,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::set< TriGlbIdx > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); std::set< TriGlbIdx > receivedIndices; // The graph nodes that my neighbors will send me. { - std::set< TriGlbIdx > tmp( std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ) ); + std::set< TriGlbIdx > const tmp( std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ) ); std::set_difference( std::cbegin( allNeededIndices ), std::cend( allNeededIndices ), std::cbegin( tmp ), std::cend( tmp ), std::inserter( receivedIndices, std::end( receivedIndices ) ) ); From 025a1322cfb6198e544181ce90b2d69daa4df946 Mon Sep 17 00:00:00 2001 From: TotoGaz <49004943+TotoGaz@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:55:56 -0700 Subject: [PATCH 100/106] Micro c++ improvements and renamings. --- .../mesh/generators/BuildPods.cpp | 66 ++++++++----------- .../mesh/generators/NewGhosting.cpp | 46 ++++++------- 2 files changed, 52 insertions(+), 60 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index f43fdf4d874..6a00f09ef24 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -506,44 +506,34 @@ void buildPods( MeshGraph const & owned, DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); - NodeMgrImpl nodeMgr = makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), - graph.n2pos, - upwardMappings.n2e, - upwardMappings.n2f, - upwardMappings.n2c, - g2l.nodes, - invertGhostSend( send.nodes, g2l.nodes ), - invertGhostRecv( recv.nodes, g2l.nodes ) ); - - EdgeMgrImpl edgeMgr = makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), - downwardMappings.e2n, - upwardMappings.e2f, - g2l.edges, - invertGhostSend( send.edges, g2l.edges ), - invertGhostRecv( recv.edges, g2l.edges ) ); - - FaceMgrImpl faceMgr = makeFlavorlessFaceMgrImpl( std::size( g2l.faces ), - downwardMappings.f2n, - downwardMappings.f2e, - upwardMappings.f2c, - g2l.faces, - invertGhostSend( send.faces, g2l.faces ), - invertGhostRecv( recv.faces, g2l.faces ) ); - - CellBlkImpl cellBlock = makeFlavorlessCellBlkImpl( std::size( g2l.cells ), - downwardMappings.c2n, - downwardMappings.c2e, - downwardMappings.c2f, - g2l.cells, - invertGhostSend( send.cells, g2l.cells ), - invertGhostRecv( recv.cells, g2l.cells ) ); - - CellMgrImpl cellMgr( std::move( cellBlock ) ); - - meshMappings.setCellMgr( std::move( cellMgr ) ); - meshMappings.setEdgeMgr( std::move( edgeMgr ) ); - meshMappings.setFaceMgr( std::move( faceMgr ) ); - meshMappings.setNodeMgr( std::move( nodeMgr ) ); + meshMappings.setEdgeMgr( makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), + downwardMappings.e2n, + upwardMappings.e2f, + g2l.edges, + invertGhostSend( send.edges, g2l.edges ), + invertGhostRecv( recv.edges, g2l.edges ) ) ); + meshMappings.setFaceMgr( makeFlavorlessFaceMgrImpl( std::size( g2l.faces ), + downwardMappings.f2n, + downwardMappings.f2e, + upwardMappings.f2c, + g2l.faces, + invertGhostSend( send.faces, g2l.faces ), + invertGhostRecv( recv.faces, g2l.faces ) ) ); + meshMappings.setNodeMgr( makeFlavorlessNodeMgrImpl( std::size( g2l.nodes ), + graph.n2pos, + upwardMappings.n2e, + upwardMappings.n2f, + upwardMappings.n2c, + g2l.nodes, + invertGhostSend( send.nodes, g2l.nodes ), + invertGhostRecv( recv.nodes, g2l.nodes ) ) ); + meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpl( std::size( g2l.cells ), + downwardMappings.c2n, + downwardMappings.c2e, + downwardMappings.c2f, + g2l.cells, + invertGhostSend( send.cells, g2l.cells ), + invertGhostRecv( recv.cells, g2l.cells ) ) ) ); meshMappings.setNeighbors( getNeighbors( recv, send ) ); } diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 1ea1af1cc31..48b9f635242 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -81,8 +81,6 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, EdgeGlbIdx const & maxEdgeId, FaceGlbIdx const & maxFaceId ) { - MaxGlbIdcs offsets{ NodeGlbIdx{ 0 }, maxEdgeId, maxFaceId, CellGlbIdx{ 0 } }; - auto const extract = []( vtkDataArray * globalIds ) -> vtkIdType { vtkIdTypeArray * gids = vtkIdTypeArray::FastDownCast( globalIds ); @@ -90,8 +88,10 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, return *std::max_element( s.begin(), s.end() ); }; - offsets.nodes = NodeGlbIdx{ extract( mesh->GetPointData()->GetGlobalIds() ) }; - offsets.cells = CellGlbIdx{ extract( mesh->GetCellData()->GetGlobalIds() ) }; + MaxGlbIdcs const offsets{ NodeGlbIdx{ extract( mesh->GetPointData()->GetGlobalIds() ) }, + maxEdgeId, + maxFaceId, + CellGlbIdx{ extract( mesh->GetCellData()->GetGlobalIds() ) } }; // Otherwise, use `MPI_Type_create_struct`. static_assert( std::is_same_v< NodeGlbIdx::UnderlyingType, EdgeGlbIdx::UnderlyingType > ); @@ -266,7 +266,9 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > } } - return { std::move( owned ), std::move( present ) }; + GEOS_ASSERT( std::empty( present.c2f ) ); + + return { owned, present }; } //using TriLocIdx = fluent::NamedType< localIndex, struct TagTriLocIdx, fluent::Arithmetic, fluent::ImplicitlyConvertibleTo >; @@ -491,7 +493,7 @@ struct Adjacency { std::vector< TriGlbIdx > ownedNodesIdcs; std::vector< TriGlbIdx > ownedGlbIdcs; - std::vector< TriGlbIdx > otherGlbIdcs; + std::vector< TriGlbIdx > presentGlbIdcs; Teuchos::Array< std::size_t > numEntriesPerRow; std::vector< std::vector< TriGlbIdx > > indices; std::vector< std::vector< TriScalarInt > > values; @@ -509,32 +511,32 @@ Adjacency buildAdjacency( MeshGraph const & owned, // Aliases std::vector< TriGlbIdx > & ownedNodesIdcs = adjacency.ownedNodesIdcs; std::vector< TriGlbIdx > & ownedGlbIdcs = adjacency.ownedGlbIdcs; - std::vector< TriGlbIdx > & otherGlbIdcs = adjacency.otherGlbIdcs; + std::vector< TriGlbIdx > & presentGlbIdcs = adjacency.presentGlbIdcs; Teuchos::Array< std::size_t > & numEntriesPerRow = adjacency.numEntriesPerRow; std::vector< std::vector< TriGlbIdx > > & indices = adjacency.indices; std::vector< std::vector< TriScalarInt > > & values = adjacency.values; ownedNodesIdcs.reserve( std::size( owned.n2pos ) ); ownedGlbIdcs.reserve( numOwned ); - otherGlbIdcs.reserve( numOther ); + presentGlbIdcs.reserve( numOther ); numEntriesPerRow.reserve( numOwned ); indices.reserve( numOwned ); values.reserve( numOwned ); for( auto const & [ngi, _]: present.n2pos ) { - otherGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); + presentGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); } for( auto const & [egi, _]: present.e2n ) { - otherGlbIdcs.emplace_back( convert.fromEdgeGlbIdx( egi ) ); + presentGlbIdcs.emplace_back( convert.fromEdgeGlbIdx( egi ) ); } for( auto const & [fgi, _]: present.f2e ) { - otherGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); + presentGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); } - std::sort( std::begin( otherGlbIdcs ), std::end( otherGlbIdcs ) ); - GEOS_ASSERT_EQ( numOther, std::size( otherGlbIdcs ) ); + std::sort( std::begin( presentGlbIdcs ), std::end( presentGlbIdcs ) ); + GEOS_ASSERT_EQ( numOther, std::size( presentGlbIdcs ) ); for( auto const & [ngi, _]: owned.n2pos ) { @@ -644,12 +646,12 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & Adjacency const adjacency = buildAdjacency( owned, present, convert ); std::size_t const numOwned = std::size( adjacency.ownedGlbIdcs ); - std::size_t const numOther = std::size( adjacency.otherGlbIdcs ); + std::size_t const numPresent = std::size( adjacency.presentGlbIdcs ); // Aliases std::vector< TriGlbIdx > const & ownedNodesIdcs = adjacency.ownedNodesIdcs; std::vector< TriGlbIdx > const & ownedGlbIdcs = adjacency.ownedGlbIdcs; - std::vector< TriGlbIdx > const & otherGlbIdcs = adjacency.otherGlbIdcs; + std::vector< TriGlbIdx > const & presentGlbIdcs = adjacency.presentGlbIdcs; Teuchos::Array< std::size_t > const & numEntriesPerRow = adjacency.numEntriesPerRow; std::vector< std::vector< TriGlbIdx > > const & indices = adjacency.indices; std::vector< std::vector< TriScalarInt > > const & values = adjacency.values; @@ -682,11 +684,11 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // It contains one for every time the geometrical entity is present on the rank. // By present, we do not meant that the ranks owns it: just that it's already available on the rank. auto const mpiMap = make_rcp< TriMap const >( Tpetra::global_size_t( commSize ), TriGlbIdx{ 0 }, comm ); // Let the current rank get the appropriate index in this map. - TriCrsMatrix indicator( mpiMap, numOwned + numOther ); + TriCrsMatrix indicator( mpiMap, numOwned + numPresent ); - std::vector< TriScalarInt > const ones( std::max( { numOwned, numOther, std::size_t( 1 ) } ), 1 ); + std::vector< TriScalarInt > const ones( std::max( { numOwned, numPresent, std::size_t( 1 ) } ), 1 ); indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numOwned ), ones.data(), ownedGlbIdcs.data() ); - indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numOther ), ones.data(), otherGlbIdcs.data() ); + indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numPresent ), ones.data(), presentGlbIdcs.data() ); indicator.fillComplete( ownedMap, mpiMap ); // The `ownership` matrix is a diagonal square matrix. @@ -834,7 +836,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } std::vector< TriGlbIdx > notPresentIndices; // The graphs nodes that are nor owned neither present by/on the current rank. std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), - std::cbegin( otherGlbIdcs ), std::cend( otherGlbIdcs ), + std::cbegin( presentGlbIdcs ), std::cend( presentGlbIdcs ), std::back_inserter( notPresentIndices ) ); std::vector< TriGlbIdx > notPresentNodes; std::copy_if( std::cbegin( notPresentIndices ), std::cend( notPresentIndices ), @@ -937,8 +939,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & auto const missingNodesMap = make_rcp< TriMap const >( numGlbMissingNodes, numLocMissingNodes, TriGlbIdx{ 0 }, comm ); // Following information is needed to compute the global row. - TriGlbIdx const missingIndicesOffset = missingIndicesMap->getGlobalElement( TriLocIdx{ 0 } ); // TODO Hocus Pocus - TriGlbIdx const missingNodesOffset = missingNodesMap->getGlobalElement( TriLocIdx{ 0 } ); // TODO Hocus Pocus + TriGlbIdx const missingIndicesOffset = missingIndicesMap->getGlobalElement( TriLocIdx{ 0 } ); // Let trilinos provide the offset. + TriGlbIdx const missingNodesOffset = missingNodesMap->getGlobalElement( TriLocIdx{ 0 } ); // Let trilinos provide the offset. // Indicator matrix for the all the missing quantities (nodes, edges, faces and cells). TriCrsMatrix missingIndicator( missingIndicesMap, 1 ); @@ -1125,7 +1127,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // GEOS_LOG_RANK( "ghosts_n2ps = " << json( ghosts.n2pos ) ); // } - return { std::move( ghosts ), std::move( recv ), std::move( send ) }; + return { ghosts, recv, send }; } void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, From 7058cb38476b93066f3c18be08630835d84f590b Mon Sep 17 00:00:00 2001 From: Ryan Aronson Date: Wed, 3 Jul 2024 10:54:04 -0700 Subject: [PATCH 101/106] Comments/Documentation for New Ghosting Architecture (#3189) * added comments for global numbering (Ryan's version) * comments for constructing owned and present mesh data (Ryan's version) * comments for construction of alocal djacency matrix data by each rank (Ryan's version) * Comments through ghostingFootprint (Ryan's version) * Comments through construction of GhostSend/GhostRecv for each rank (Ryan's version) * complete comments for performGhosting (Ryan's version) * Update comments about sorting following fix * finished comments of doTheNewGhosting (Ryan's version) * make new ghosting active/deactive by setting command line flag (-g 1) --- .../common/initializeEnvironment.hpp | 3 + .../mainInterface/ProblemManager.cpp | 20 +- .../mainInterface/ProblemManager.hpp | 7 +- .../mainInterface/initialization.cpp | 7 + .../mesh/generators/BuildPods.cpp | 37 +- .../mesh/generators/NewGhosting.cpp | 370 ++++++- .../mesh/generators/NewGlobalNumbering.cpp | 986 +++++++++--------- 7 files changed, 902 insertions(+), 528 deletions(-) diff --git a/src/coreComponents/common/initializeEnvironment.hpp b/src/coreComponents/common/initializeEnvironment.hpp index 044b1977294..b91ac24dc80 100644 --- a/src/coreComponents/common/initializeEnvironment.hpp +++ b/src/coreComponents/common/initializeEnvironment.hpp @@ -88,6 +88,9 @@ struct CommandLineOptions /// Print memory usage in data repository real64 printMemoryUsage = -1.0; + + /// Use new ghosting strategy + integer useNewGhosting = 0; }; /** diff --git a/src/coreComponents/mainInterface/ProblemManager.cpp b/src/coreComponents/mainInterface/ProblemManager.cpp index 631ec0efb4c..564249ae166 100644 --- a/src/coreComponents/mainInterface/ProblemManager.cpp +++ b/src/coreComponents/mainInterface/ProblemManager.cpp @@ -142,11 +142,11 @@ ProblemManager::ProblemManager( conduit::Node & root ): setRestartFlags( RestartFlags::WRITE ). setDescription( "Whether to disallow using pinned memory allocations for MPI communication buffers." ); - m_useNewGhosting = 1; -// registerWrapper( viewKeysStruct::useNewGhostingString(), &m_useNewGhosting ). -// setInputFlag( InputFlags::OPTIONAL ). -// setApplyDefaultValue( 0 ). -// setDescription( "Controls the use of the new ghosting implementation." ); + commandLine.registerWrapper< integer >( viewKeys.useNewGhosting.key( ) ). + setApplyDefaultValue( 0 ). + setRestartFlags( RestartFlags::WRITE ). + setDescription( "Whether to use new ghosting strategy" ); + } ProblemManager::~ProblemManager() @@ -191,6 +191,7 @@ void ProblemManager::parseCommandLineInput() commandLine.getReference< integer >( viewKeys.overridePartitionNumbers ) = opts.overridePartitionNumbers; commandLine.getReference< integer >( viewKeys.useNonblockingMPI ) = opts.useNonblockingMPI; commandLine.getReference< integer >( viewKeys.suppressPinned ) = opts.suppressPinned; + commandLine.getReference< integer >( viewKeys.useNewGhosting ) = opts.useNewGhosting; string & outputDirectory = commandLine.getReference< string >( viewKeys.outputDirectory ); outputDirectory = opts.outputDirectory; @@ -633,10 +634,12 @@ void ProblemManager::generateMesh() GEOS_MARK_FUNCTION; DomainPartition & domain = getDomainPartition(); + Group const & commandLine = getGroup< Group >( groupKeys.commandLine ); + MeshManager & meshManager = this->getGroup< MeshManager >( groupKeys.meshManager ); - GEOS_LOG_RANK( "m_useNewGhosting = " << m_useNewGhosting ); - meshManager.generateMeshes( m_useNewGhosting, domain ); + GEOS_LOG_RANK( "useNewGhosting = " << commandLine.getReference< integer >( viewKeys.useNewGhosting ) ); + meshManager.generateMeshes( commandLine.getReference< integer >( viewKeys.useNewGhosting ), domain ); // get all the discretizations from the numerical methods. // map< pair< mesh body name, pointer to discretization>, array of region names > @@ -658,7 +661,7 @@ void ProblemManager::generateMesh() } else { - if( m_useNewGhosting ) + if( commandLine.getReference< integer >( viewKeys.useNewGhosting ) ) { GEOS_LOG_RANK_0( "Generating the mesh levels for the new ghosting." ); generateMeshLevelFreeFct( meshBody.getMeshMappings(), @@ -685,7 +688,6 @@ void ProblemManager::generateMesh() } } ); - Group const & commandLine = this->getGroup< Group >( groupKeys.commandLine ); integer const useNonblockingMPI = commandLine.getReference< integer >( viewKeys.useNonblockingMPI ); // domain.setupBaseLevelMeshGlobalInfo(); diff --git a/src/coreComponents/mainInterface/ProblemManager.hpp b/src/coreComponents/mainInterface/ProblemManager.hpp index ed4635a7d4d..7a201463085 100644 --- a/src/coreComponents/mainInterface/ProblemManager.hpp +++ b/src/coreComponents/mainInterface/ProblemManager.hpp @@ -230,9 +230,7 @@ class ProblemManager : public dataRepository::Group dataRepository::ViewKey outputDirectory = {"outputDirectory"}; ///< Output directory key dataRepository::ViewKey useNonblockingMPI = {"useNonblockingMPI"}; ///< Flag to use non-block MPI key dataRepository::ViewKey suppressPinned = {"suppressPinned"}; ///< Flag to suppress use of pinned memory key - constexpr static char const * useNewGhostingString() - { return "useNewGhosting"; } - + dataRepository::ViewKey useNewGhosting = {"useNewGhosting"}; ///< Flag to use new ghosting strategies } viewKeys; ///< Command line input viewKeys /// Child group viewKeys @@ -385,9 +383,6 @@ class ProblemManager : public dataRepository::Group /// The FieldSpecificationManager FieldSpecificationManager * m_fieldSpecificationManager; - - /// Whether we should use the new ghosting implementation. - integer m_useNewGhosting = 0; }; } /* namespace geos */ diff --git a/src/coreComponents/mainInterface/initialization.cpp b/src/coreComponents/mainInterface/initialization.cpp index 75ea5de3b61..0d28cfaf534 100644 --- a/src/coreComponents/mainInterface/initialization.cpp +++ b/src/coreComponents/mainInterface/initialization.cpp @@ -103,6 +103,7 @@ std::unique_ptr< CommandLineOptions > parseCommandLineOptions( int argc, char * TRACE_DATA_MIGRATION, MEMORY_USAGE, PAUSE_FOR, + NEW_GHOST, }; const option::Descriptor usage[] = @@ -123,6 +124,7 @@ std::unique_ptr< CommandLineOptions > parseCommandLineOptions( int argc, char * { TRACE_DATA_MIGRATION, 0, "", "trace-data-migration", Arg::None, "\t--trace-data-migration, \t Trace host-device data migration" }, { MEMORY_USAGE, 0, "m", "memory-usage", Arg::nonEmpty, "\t-m, --memory-usage, \t Minimum threshold for printing out memory allocations in a member of the data repository." }, { PAUSE_FOR, 0, "", "pause-for", Arg::numeric, "\t--pause-for, \t Pause geosx for a given number of seconds before starting execution" }, + { NEW_GHOST, 0, "g", "new-ghosting", Arg::numeric, "\t-g, --new-ghosting, \t Use new ghosting strategy" }, { 0, 0, nullptr, nullptr, nullptr, nullptr } }; @@ -237,6 +239,11 @@ std::unique_ptr< CommandLineOptions > parseCommandLineOptions( int argc, char * std::this_thread::sleep_for( std::chrono::seconds( duration ) ); } break; + case NEW_GHOST: + { + commandLineOptions->useNewGhosting = 1; + } + break; } } diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 6a00f09ef24..acb25238665 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -55,11 +55,14 @@ std::map< GI, toLocIdx_t< GI > > buildGlobalToLocalMap( std::set< GI > const & g return g2l; } +// Consolidate the mesh graphs on the current rank into one object MeshGraph mergeMeshGraph( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts ) { + // inititalize the consolidated version with owned MeshGraph result{ owned }; + // insert the present and ghost data for( MeshGraph const & graph: { present, ghosts } ) { result.c2f.insert( std::cbegin( graph.c2f ), std::cend( graph.c2f ) ); @@ -89,6 +92,7 @@ struct UpwardMappings std::map< NodeLocIdx, std::vector< CellLocIdx > > n2c; }; +// converts generic map to arrayOfArrays template< class T, class U > ArrayOfArrays< localIndex > convertToAoA( std::map< T, std::vector< U > > const & t2u ) { @@ -138,6 +142,7 @@ array2d< localIndex, P > convertToA2d( std::map< T, std::vector< U > > const & t return t2u_; } +// converts map to unordered map template< class GI > unordered_map< globalIndex, localIndex > convertGlobalToLocalMap( std::map< GI, toLocIdx_t< GI > > const & g2l ) { @@ -150,6 +155,7 @@ unordered_map< globalIndex, localIndex > convertGlobalToLocalMap( std::map< GI, return g2l_; } +// removes MPIRank typed int, changes from std::vector to array1d template< typename LI > std::map< integer, array1d< localIndex > > toFlavorlessMapping( std::map< MpiRank, std::vector< LI > > const & input ) { @@ -165,7 +171,7 @@ std::map< integer, array1d< localIndex > > toFlavorlessMapping( std::map< MpiRan } return output; -}; +} EdgeMgrImpl makeFlavorlessEdgeMgrImpl( std::size_t const & numEdges, std::map< EdgeLocIdx, std::tuple< NodeLocIdx, NodeLocIdx > > const & e2n, @@ -219,6 +225,8 @@ FaceMgrImpl makeFlavorlessFaceMgrImpl( std::size_t const & numFaces, toFlavorlessMapping( recv ) ); } +// Function to create a nodeManager from our local upward mapping data +// By flavorless, we are indicating that we are changing data-structures to GEOS ones, etc NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, std::map< NodeGlbIdx, std::array< double, 3 > > const & n2pos, std::map< NodeLocIdx, std::vector< EdgeLocIdx > > const & n2e, @@ -282,6 +290,7 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, DownwardMappings res; // Building the `e2n` (edges to nodes) mapping + // Simply looping through the meshgraph and converting to local indices for( auto const & [egi, ngis]: graph.e2n ) { NodeLocIdx const nli0 = g2l.nodes.at( std::get< 0 >( ngis ) ); @@ -290,8 +299,10 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, } // Building the `f2n` (face to nodes) and `f2e` (faces to edges) mappings + // Loop through meshgraph for( auto const & [fgi, edgeInfos]: graph.f2e ) { + // convert to face local index FaceLocIdx const & fli = g2l.faces.at( fgi ); std::vector< NodeLocIdx > & nodes = res.f2n[fli]; @@ -300,18 +311,23 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, nodes.reserve( std::size( edgeInfos ) ); edges.reserve( std::size( edgeInfos ) ); + // Loop over the edges connected to the face for( EdgeInfo const & edgeInfo: edgeInfos ) { + // use the meshgraph again to go from edge to node, convert to local indices to fill face2node std::tuple< NodeGlbIdx, NodeGlbIdx > const & ngis = graph.e2n.at( edgeInfo.index ); nodes.emplace_back( edgeInfo.start == 0 ? g2l.nodes.at( std::get< 0 >( ngis ) ) : g2l.nodes.at( std::get< 1 >( ngis ) ) ); + // just convert edge global id to local to fill face2edge edges.emplace_back( g2l.edges.at( edgeInfo.index ) ); } } // Building the `c2n` (cell to nodes), `c2e` (cell to edges) and `c2f` (cell to faces) mappings + // loop thorugh meshgraph cells2face for( auto const & [cgi, faceInfos]: graph.c2f ) { + // convert to cell local index CellLocIdx const & cli = g2l.cells.at( cgi ); std::vector< FaceLocIdx > & faces = res.c2f[cli]; @@ -321,12 +337,14 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, edges.reserve( std::size( faceInfos ) ); // c2f + // simple conversion of global to local index is all thats needed for( FaceInfo const & faceInfo: faceInfos ) { faces.emplace_back( g2l.faces.at( faceInfo.index ) ); } // c2e + // loop over the faces in the cell and collect all their edges std::set< EdgeLocIdx > tmpEdges; for( FaceLocIdx const & fli: faces ) { @@ -336,16 +354,19 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, edges.assign( std::cbegin( tmpEdges ), std::cend( tmpEdges ) ); // c2n + // Note how we are hard-coded for hexs FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type. FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7) std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( g2l.faces.at( bottomFace.index ) ); std::vector< NodeLocIdx > const & topNodes = res.f2n.at( g2l.faces.at( topFace.index ) ); + // reserFaceNodes resets the order using the flipped and start info std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); // TODO carefully check the ordering... + // looks like this is geos order std::array< NodeLocIdx, 8 > const tmp{ bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; res.c2n[cli] = { tmp[0], tmp[1], tmp[3], tmp[2], tmp[4], tmp[5], tmp[7], tmp[6] }; } @@ -494,8 +515,12 @@ void buildPods( MeshGraph const & owned, GhostSend const & send, MeshMappingImpl & meshMappings ) { + // start by merging the 3 mesh graphs for the different types of data on the graph into 1 struct MeshGraph const graph = mergeMeshGraph( owned, present, ghosts ); + // create GlobalToLocal, which maps global IDs to rank local IDs for (nodes, edges, faces, cells) + // mapKeys is just a utility which extracts keys + // buildGlobalToLocalMap just takes the global ids and maps them to 1, 2, 3, ... GlobalToLocal const g2l{ buildGlobalToLocalMap( mapKeys< std::set >( graph.n2pos ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.e2n ) ), @@ -503,9 +528,19 @@ void buildPods( MeshGraph const & owned, buildGlobalToLocalMap( mapKeys< std::set >( graph.c2f ) ) }; + // Now we build the mappings used by GEOS + // Downward - edges2nodes, faces2edges, faces2nodes, cells2faces, cells2edges, cells2nodes + // The difference is that these use local indices, thus we built g2l + // there are assumptions that we are working on hexs here DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); + // invert to downward mappings to get e2f, f2c, n2e, n2f, n2c UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); + // Now we can make our node/edge/face/cell manager + // NodeMgrImpl inherits from nodeManager (see pods.hpp) (interface with getters and setters) + // We basically have all the data, this function just casts it into the proper GEOS types + // same for edges, faces, cells + meshMappings.setEdgeMgr( makeFlavorlessEdgeMgrImpl( std::size( g2l.edges ), downwardMappings.e2n, upwardMappings.e2f, diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 48b9f635242..953801b6541 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -88,11 +88,17 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, return *std::max_element( s.begin(), s.end() ); }; + // MaxGlbIdcs is just a struct of the 4 index types (ints under the hood) + // edge and face max (global) index values are passed in, + // we use a little lambda to get the node and cell max indices MaxGlbIdcs const offsets{ NodeGlbIdx{ extract( mesh->GetPointData()->GetGlobalIds() ) }, maxEdgeId, maxFaceId, CellGlbIdx{ extract( mesh->GetCellData()->GetGlobalIds() ) } }; + // I would have to look in detail about what these MPI lines do, + // But im fairly sure they are just handling type stuff so that you can do communications + // with the MaxGlbIdcs type // Otherwise, use `MPI_Type_create_struct`. static_assert( std::is_same_v< NodeGlbIdx::UnderlyingType, EdgeGlbIdx::UnderlyingType > ); static_assert( std::is_same_v< NodeGlbIdx::UnderlyingType, FaceGlbIdx::UnderlyingType > ); @@ -107,6 +113,8 @@ MaxGlbIdcs gatherOffset( vtkSmartPointer< vtkDataSet > mesh, MPI_Op op; MPI_Op_create( g, true, &op ); + // Do an allReduce so that every rank knows the max index for (nodes, edges, faces, cells) + // g is the custom reduction function which takes max for each (nodes, edges, faces, cells) MaxGlbIdcs result( offsets ); MPI_Allreduce( &offsets, &result, 1, t, op, MPI_COMM_WORLD ); @@ -167,13 +175,25 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > BucketOffsets const & offsets, MpiRank curRank ) { + // MeshGraph (defined in BuildPods.hpp) is just a struct of maps which map: + // cell global index to vector of entries of type (face global index, extra stuff defining orientation of face) for each of the adjacent faces + // face global index to vector of entries of type (edge global index plus orientation) for each of the adjacent edges + // edge global index to tuple of node global indices + // node global index to 3d array of position + // We distinguish between the owned data and the present data + // present data is stuff that is owned by another rank, but is already present on the current rank because of the boundary exchange done previously. + // Note that present just includes the 2D boundary between ranks, we still need the cell info, the other faces, edges, nodes which make up that cell + // (and further if we wanted deeper ghosting) MeshGraph owned, present; + // Any (node, edge, face, cell) is owned by the lowest rank on which it appears + // so we define a lambda to compute if the current rank owns some piece of data auto const isCurrentRankOwning = [&curRank]( std::set< MpiRank > const & ranks ) -> bool { return curRank == *std::min_element( std::cbegin( ranks ), std::cend( ranks ) ); }; + // Get the nodal positions for nodes on current rank mesh and store in appropriate MeshGraph (owned/present) std::map< NodeGlbIdx, vtkIdType > const n2vtk = buildNgiToVtk( mesh ); for( auto const & [ranks, nodes]: buckets.nodes ) { @@ -185,10 +205,13 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > } } + // Loop through edge data and assign to the owned/present graph for( auto const & [ranks, edges]: buckets.edges ) { auto & e2n = isCurrentRankOwning( ranks ) ? owned.e2n : present.e2n; - EdgeGlbIdx egi = offsets.edges.at( ranks ); // TODO hack + // im not sure why this is a 'hack', this seems like the way I would do it + // namely get the edge offset for this bucket, then count upwards in global Id for each edge in bucket + EdgeGlbIdx egi = offsets.edges.at( ranks ); // TODO hack for( Edge const & edge: edges ) { e2n[egi] = edge; @@ -203,33 +226,40 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > e2n.insert( std::cbegin( m ), std::cend( m ) ); } - // Simple inversion + // Simple inversion of the e2n (which didnt care about ownership) to a node2edge (which therefore also doesnt care about ownership) + // this is used in constructing the face2edge data below std::map< std::tuple< NodeGlbIdx, NodeGlbIdx >, EdgeGlbIdx > n2e; for( auto const & [e, n]: e2n ) { n2e[n] = e; // TODO what about ownership? } + // Loop through the face data to assign to owned/present graph for( auto const & [ranks, faces]: buckets.faces ) { auto & f2e = isCurrentRankOwning( ranks ) ? owned.f2e : present.f2e; + // grab initial index for faces in this bucket and then loop over these faces FaceGlbIdx fgi = offsets.faces.at( ranks ); for( Face face: faces ) // Intentional copy for the future `emplace_back`. { - face.emplace_back( face.front() ); // Trick to build the edges. + face.emplace_back( face.front() ); // Trick to build the edges. Namely you need to build the edge that connects the last node to the first for( std::size_t i = 0; i < face.size() - 1; ++i ) { + // An edge is just the pair of node global indices, plus a boolean telling you the order + // This is placed into an EdgeInfo struct (defined in buildPods.hpp, but its just this info) NodeGlbIdx const & n0 = face[i], & n1 = face[i + 1]; std::pair< NodeGlbIdx, NodeGlbIdx > const p0 = std::make_pair( n0, n1 ); std::pair< NodeGlbIdx, NodeGlbIdx > const p1 = std::minmax( n0, n1 ); - EdgeInfo const info = { n2e.at( p1 ), std::uint8_t{ p0 != p1 } }; - f2e[fgi].emplace_back( info ); + EdgeInfo const info = { n2e.at( p1 ), std::uint8_t{ p0 != p1 } }; // n2e is used to get the edge global index from the node global indices + f2e[fgi].emplace_back( info ); // assign data to proper graph } - ++fgi; + ++fgi; // increment the offset for the bucket } } + // create a map from node global indices to face global index which does not care about ownership, similar to n2e above + // this will be used in generating cell2face data, like how n2e was used in generating face2edge std::map< std::vector< NodeGlbIdx >, FaceGlbIdx > n2f; for( auto const & [ranks, faces]: buckets.faces ) { @@ -250,6 +280,7 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > // TODO copy paste? for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) { + // for each of the faces in the cell, get the global ids of the nodes vtkCell * face = cell->GetFace( f ); vtkIdList * pids = face->GetPointIds(); std::vector< NodeGlbIdx > faceNodes( pids->GetNumberOfIds() ); @@ -259,9 +290,17 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > vtkIdType const ngi = globalPtIds->GetValue( nli ); faceNodes[i] = NodeGlbIdx{ ngi }; } + + // these nodes will be in the vtk ordering, we want a consistent ordering for each face + // reorderFaceNodes (defined in NewGlobalNumbering) does this by starting with the smallest node index, + // and then iterating in the direction of the smaller of its neighbors bool isFlipped; std::uint8_t start; std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes, isFlipped, start ); + + // add the face data to the vector for this cell in owned graph. + // note there will never be any 'present' cells, as the earlier exchange only communicates the 2D boundary btwn ranks + // note we are using the n2f built above to go from the vector of nodes (properly ordered) to the face global ID owned.c2f[gci].emplace_back( FaceInfo{ n2f.at( reorderedFaceNodes ), isFlipped, start } ); } } @@ -322,13 +361,27 @@ TriCrsMatrix multiply( int commSize, Teuchos::RCP< TriMap const > ownedMap = upward.getRowMap(); Teuchos::RCP< TriMap const > mpiMap = indicator.getRangeMap(); - // Upward (n -> e -> f -> c) + // row j of indicator was just ones in columns corresponding to present entities on rank j + // row i of upward was constructed as having nonzeros in the columns of entities on which the entity corresponding to row i directly depended + // Begin graph traversal by matrix matrix multiplying + // Trilinos matrix matrix multiply https://docs.trilinos.org/dev/packages/tpetra/doc/html/namespaceTpetra_1_1MatrixMatrix.html#a4c3132fa56ba6d5b071f053486529111 + + // Upward (n -> e -> f -> c) + + // u0_0 (num_entities x num_rank) = upward (num_entities x num_entities) * indicator^T (num_entities x num_rank) + // Using the `upward` interpretation, multiplying upward by each column of indicator^T would give + // a column vector noting the all the faces in the global mesh which contain the edges owned by that rank, all the global cells which contain the owned faces, etc + // These are the colums of the result `result_u0_0` TriCrsMatrix result_u0_0( ownedMap, commSize ); Tpetra::MatrixMatrix::Multiply( upward, false, indicator, true, result_u0_0, false ); result_u0_0.fillComplete( mpiMap, ownedMap ); Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-0.mat", result_u0_0 ); + // We repeat this process to make sure we catch all the dependencies, but im not convinced once isnt enough + // In theory, you iterate until the matrix stops changing + // TODO: test this hypothesis + // TODO: you might also be able to just do one multiplication with upward + upward^T to do everything at once, but perhaps decoding becomes an issue TriCrsMatrix result_u0_1( ownedMap, commSize ); Tpetra::MatrixMatrix::Multiply( upward, false, result_u0_0, false, result_u0_1, false ); result_u0_1.fillComplete( mpiMap, ownedMap ); @@ -339,8 +392,14 @@ TriCrsMatrix multiply( int commSize, result_u0_2.fillComplete( mpiMap, ownedMap ); Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/result-2.mat", result_u0_2 ); - // Downward (c -> f -> e -> n) + // Now we know, for each rank, all the geometrical quantities in the global mesh which include something from that rank + // Downward (c -> f -> e -> n) + // Note that we still need to do a traversal in the "opposite direction" + // Consider starting with some edge, and applying the process above. + // You will recover the faces containing it, and the cells which contain those + // What you wont get, for example, is any other (faces, edges, nodes) in that cell + // Thus we repeat the process with downward = upward^T TriCrsMatrix result_d0_0( ownedMap, commSize ); Tpetra::MatrixMatrix::Multiply( upward, true, result_u0_2, false, result_d0_0, false ); result_d0_0.fillComplete( mpiMap, ownedMap ); @@ -358,9 +417,16 @@ TriCrsMatrix multiply( int commSize, MpiWrapper::barrier(); + // Now we know, for each rank, the set of all global geometric entities which are included by any geometric entity that includes something from that rank + // (what a mouthful) + // Note that we have NOT made a distinction between present and owned on the rank though + return result_d0_2; } +// Helper class which converts between matrix indices and (node, edge, face, cell) indices. +// The matrix rows/cols represent all the nodes, then all the edges, then all the faces, then all the cells +// so conversion just means adding/subtracting the proper offsets class FindGeometricalType { public: @@ -454,6 +520,14 @@ class FindGeometricalType TriGlbIdx const m_numEntries; }; +// Helper function to encode geometrical information into ints which can be entered into adjacency matrix +// For example, a face is connected to multiple edges, so in the face row, there will be a nonzero entry in the col for each edge +// But if we just put a 1 in each we lose the information like what order the edges were in and what order the nodes were in +// So instead we put in a value such that we can reconstruct this info if needed later +// For encoding the edges within a face, N = 2, basis = NumEdges, array = {start node (0,1), index of edge within face}, +// so that result = start_node*numEdges+index_in_face +// For encoding faces within a cell, N = 3, basis = numFaces, array = { isFlipped, start_index, index of face in cell }, +// so that result = (isFlipped*numFaces + start_index)*numFaces + index_in_cell template< std::uint8_t N > std::size_t encode( std::size_t const & basis, std::array< std::size_t, N > const & array ) @@ -489,6 +563,7 @@ std::array< std::size_t, N > decode( std::size_t const & basis, return result; } +// Structure which describes the adjacency matrix for the mesh graph with the data on current rank struct Adjacency { std::vector< TriGlbIdx > ownedNodesIdcs; @@ -499,16 +574,20 @@ struct Adjacency std::vector< std::vector< TriScalarInt > > values; }; +// Function to populate the adjacency struct defined directly above +// Note that we really build I+A, as in the end we will essentially be doing graph traversal by multiplying by (I+A)^n, ((I+A)^T)^n +// Including I gives you everything in the graph 'up to' n steps away, without it you only get things 'exactly' n steps away Adjacency buildAdjacency( MeshGraph const & owned, MeshGraph const & present, FindGeometricalType const & convert ) { Adjacency adjacency; + // Num matrix indices owned by the current rank, and the num indices corresponding to the 'present' data std::size_t const numOwned = std::size( owned.n2pos ) + std::size( owned.e2n ) + std::size( owned.f2e ) + std::size( owned.c2f ); std::size_t const numOther = std::size( present.n2pos ) + std::size( present.e2n ) + std::size( present.f2e ); - // Aliases + // Aliases for the data in adjacency std::vector< TriGlbIdx > & ownedNodesIdcs = adjacency.ownedNodesIdcs; std::vector< TriGlbIdx > & ownedGlbIdcs = adjacency.ownedGlbIdcs; std::vector< TriGlbIdx > & presentGlbIdcs = adjacency.presentGlbIdcs; @@ -516,6 +595,7 @@ Adjacency buildAdjacency( MeshGraph const & owned, std::vector< std::vector< TriGlbIdx > > & indices = adjacency.indices; std::vector< std::vector< TriScalarInt > > & values = adjacency.values; + // we can go ahead and set sizes for the current rank matrix data ownedNodesIdcs.reserve( std::size( owned.n2pos ) ); ownedGlbIdcs.reserve( numOwned ); presentGlbIdcs.reserve( numOther ); @@ -523,6 +603,9 @@ Adjacency buildAdjacency( MeshGraph const & owned, indices.reserve( numOwned ); values.reserve( numOwned ); + // fill the matrix index data for the 'present' gemetrical entities (nodes, edges, faces) + // Note that here we just note the indices, we dont fill any matrix entry or anything + // The filling of the matrix entries is done by the owning rank for( auto const & [ngi, _]: present.n2pos ) { presentGlbIdcs.emplace_back( convert.fromNodeGlbIdx( ngi ) ); @@ -535,9 +618,11 @@ Adjacency buildAdjacency( MeshGraph const & owned, { presentGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); } - std::sort( std::begin( presentGlbIdcs ), std::end( presentGlbIdcs ) ); - GEOS_ASSERT_EQ( numOther, std::size( presentGlbIdcs ) ); + std::sort( std::begin( presentGlbIdcs ), std::end( presentGlbIdcs ) ); // I think that the data in n2pos, e2n, f2e was not necessarily sorted, should double check this as sorting caused a bug in owned data + GEOS_ASSERT_EQ( numOther, std::size( presentGlbIdcs ) ); // ensure we added the correct amount of stuff + + // Now do owned data for( auto const & [ngi, _]: owned.n2pos ) { // Nodes depend on no other geometrical entity, @@ -557,11 +642,15 @@ Adjacency buildAdjacency( MeshGraph const & owned, // To keep the symmetry with the faces (see below), // - we store the local index (0 or 1) to express this information. // - we store the number of underlying nodes (2) on the diagonal. - size_t constexpr numNodes = std::tuple_size_v< decltype( nodes ) >; + size_t constexpr numNodes = std::tuple_size_v< decltype( nodes ) >; // should always be 2 + // This rank owns this row of the matrix ownedGlbIdcs.emplace_back( convert.fromEdgeGlbIdx( egi ) ); numEntriesPerRow.push_back( numNodes + 1 ); // `+1` comes from the diagonal + + // for the row correxponding to this edge global index, have entries in col corresponding to the + // node global indices it connects, and an entry on the diagonal indices.emplace_back( std::vector< TriGlbIdx >{ convert.fromNodeGlbIdx( std::get< 0 >( nodes ) ), convert.fromNodeGlbIdx( std::get< 1 >( nodes ) ), ownedGlbIdcs.back() } ); @@ -575,24 +664,33 @@ Adjacency buildAdjacency( MeshGraph const & owned, // Faces point to multiple edges, but their edges can be flipped w.r.t. the canonical edges // (ie minimal node global index comes first). // In order not to lose this information (held by an `EdgeInfo` instance), we serialise - // - the `EdgeInfo` instance, + // - the `EdgeInfo` instance, (recall `EdgeInfo` is just struct with edge global ID and start node (0,1), see BuildPods.hpp) // - plus the order in which the edges are appearing, // as an integer and store it as an entry in the matrix. // On the diagonal, we'll store the number of edges of the face. std::size_t const numEdges = std::size( edges ); + // This rank owns this row of the matrix ownedGlbIdcs.emplace_back( convert.fromFaceGlbIdx( fgi ) ); numEntriesPerRow.push_back( numEdges + 1 ); // `+1` comes from the diagonal + + // go ahead and add a vector of the correct size to represent the col indices and values for this row of the matrix (not yet populated) std::vector< TriGlbIdx > & ind = indices.emplace_back( numEntriesPerRow.back() ); std::vector< TriScalarInt > & val = values.emplace_back( numEntriesPerRow.back() ); + + // populate the data for this row of the matrix for( std::size_t i = 0; i < numEdges; ++i ) { + // col index is just obtained from edge global id in EdgeInfo for this edge EdgeInfo const & edgeInfo = edges[i]; ind[i] = convert.fromEdgeGlbIdx( edgeInfo.index ); + // use a single int for each edge col index to represent if the edge is flipped and position of edge in face + // in this case v = 1 + start_node*numEdges + index_in_face (again add 1 so that zero indexing becomes 1 indexing) std::size_t const v = 1 + encode< 2 >( numEdges, { edgeInfo.start, i } ); val[i] = intConv< TriScalarInt >( v ); } + // Add diagonal data ind.back() = ownedGlbIdcs.back(); val.back() = intConv< TriScalarInt >( numEdges ); } @@ -600,22 +698,33 @@ Adjacency buildAdjacency( MeshGraph const & owned, { // The same comment as for faces and edges applies for cells and faces (see above). // The main differences being that - // - the `FaceInfo` as a little more information, - // - the diagonal stores the type of the cell which conveys the number of face, nodes, ordering... + // - the `FaceInfo` as a little more information, + // (recall FaceInfo is a struct which holds the global face index, the starting node index, and boolean telling which direction the nodes should be traversed) + // - the diagonal stores the type of the cell which conveys the number of face, nodes, ordering... (note this is still a TODO, right now its just numFaces) std::size_t const numFaces = std::size( faces ); + // This rank owns this row of the matrix ownedGlbIdcs.emplace_back( convert.fromCellGlbIdx( cgi ) ); numEntriesPerRow.push_back( numFaces + 1 ); // `+1` comes from the diagonal + + // go ahead and add a vector of the correct size to represent the col indices and values for this row of the matrix (not yet populated) std::vector< TriGlbIdx > & ind = indices.emplace_back( numEntriesPerRow.back() ); std::vector< TriScalarInt > & val = values.emplace_back( numEntriesPerRow.back() ); + + // populate the data for this row of the matrix for( std::size_t i = 0; i < numFaces; ++i ) { + // col index is just gotten from the global id in FaceInfo for this face FaceInfo const & faceInfo = faces[i]; ind[i] = convert.fromFaceGlbIdx( faceInfo.index ); + + // Encode data with a single int like above, but now int tells us start_node, direction (flipped), and position of face in cell + // in this case v = 1 + (isFlipped*numFaces + start_index)*numFaces + index_in_cell (again add 1 so that zero indexing becomes 1 indexing) std::size_t const v = 1 + encode< 3 >( numFaces, { faceInfo.isFlipped, faceInfo.start, i } ); val[i] = intConv< TriScalarInt >( v ); } + // add diagonal data ind.back() = ownedGlbIdcs.back(); val.back() = intConv< TriScalarInt >( numFaces ); // TODO This should be Hex and the not the number of faces... } @@ -637,13 +746,20 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & MaxGlbIdcs const & gis, MpiRank curRank ) { + // Trilinos tools package smart reference counting pointer, see + // https://docs.trilinos.org/dev/packages/teuchos/doc/html/classTeuchos_1_1RCP.html#details + // https://docs.trilinos.org/dev/packages/teuchos/doc/html/RefCountPtrBeginnersGuideSAND.pdf using Teuchos::RCP; + // instantiate class which converts between matrix index and (node, edge, face, cell) index + // the max global indices of (node, edge, face, cell) describe where we change geometrical elements in the matrix indices, + // where all the geometrical entities are stacked one after the other FindGeometricalType const convert( gis.nodes, gis.edges, gis.faces, gis.cells ); using Geom = FindGeometricalType::Geom; std::size_t const n = convert.numEntries(); // Total number of entries in the graph. + // Build the adjacency matrix data for this rank (each rank fills in the values for geometry that it 'owns', but notes the geometry that is 'present') Adjacency const adjacency = buildAdjacency( owned, present, convert ); std::size_t const numOwned = std::size( adjacency.ownedGlbIdcs ); std::size_t const numPresent = std::size( adjacency.presentGlbIdcs ); @@ -656,45 +772,67 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::vector< std::vector< TriGlbIdx > > const & indices = adjacency.indices; std::vector< std::vector< TriScalarInt > > const & values = adjacency.values; + // Makes a Trilinos tools MPI communicator and return smart pointer to it RCP< TriComm const > const comm = make_rcp< Teuchos::MpiComm< int > const >( MPI_COMM_GEOSX ); + // Makes a const TriMap, passing the arguments to the constructor of TriMap, returns smart pointer to it + // note TriMap = Tpetra::Map< TriLocIdx, TriGlbIdx >, created using this constructor https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1Map.html#a4a21c6654fc9fb6db05644a52cc0128d + // Ultimately just collecting who owns what parts of the global matrix auto const ownedMap = make_rcp< TriMap const >( Tpetra::global_size_t( n ), ownedGlbIdcs.data(), TriLocIdx( numOwned ), TriGlbIdx{ 0 }, comm ); // The `upward` matrix offers a representation of the graph connections. // The matrix is square. Each size being the number of geometrical entities. + // The matrix is just the assembed, global adjacency. + // Note that the matrix will end up being lower triangular, and banded because edge rows only have nonzeros in the node columns, faces only have nonzeros with edges, etc + // Quick note on why we call it upward: + // Consider a vector where we put a 1 in the row corresponding to some edge with matrix index i. + // Then multiply upward times this vector. The result has nonzeros in the same row I (because we filled the diagonal of the matrix) + // and also any other rows where column i had nonzeros, which is exactly the indices of the faces containing the edge. + // Note TriCrsMatrix = Tpetra::CrsMatrix< TriScalarInt , TriLocIdx, TriGlbIdx >; https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1CrsMatrix.html TriCrsMatrix upward( ownedMap, numEntriesPerRow() ); + // Fill the global matrix rows that this rank owns + //(note that this is where the sorting things I mentioned above becomes an issue, if you reorder only the ownedGlobalIDs you assign the wrong rows) for( std::size_t i = 0; i < numOwned; ++i ) { std::vector< TriGlbIdx > const & rowIndices = indices[i]; std::vector< TriScalarInt > const & rowValues = values[i]; GEOS_ASSERT_EQ( std::size( rowIndices ), numEntriesPerRow[i] ); GEOS_ASSERT_EQ( std::size( rowValues ), numEntriesPerRow[i] ); + //https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1CrsMatrix.html#a8d4784081c59f27391ad825c4dae5e86 upward.insertGlobalValues( ownedGlbIdcs[i], TriLocIdx( std::size( rowIndices ) ), rowValues.data(), rowIndices.data() ); } + // Signal that we are done changing the values/structure of the global adjacency matrix + // https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1CrsMatrix.html#aa985b225a24d2f74602e25b38b4430af upward.fillComplete( ownedMap, ownedMap ); + + // Note this can cause some issues if /tmp doesnt exist, etc (at least it did in my codespace) Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/upward.mat", upward ); int const commSize( MpiWrapper::commSize() ); + // Now we are going to build the various other matrices which will be used in conjunction with the adjacency to ghost + // Now let's build the domain indicator matrix. - // It's rectangular, number of rows being the number of MPI ranks, - // number of columns being the number of nodes in the mesh graph, - // ie the number of geometrical entities in the mesh. + // It's rectangular, number of rows being the number of MPI ranks (so each rank builds its row), + // number of columns being the number of nodes in the mesh graph, ie the number of geometrical entities in the mesh. // It contains one for every time the geometrical entity is present on the rank. // By present, we do not meant that the ranks owns it: just that it's already available on the rank. auto const mpiMap = make_rcp< TriMap const >( Tpetra::global_size_t( commSize ), TriGlbIdx{ 0 }, comm ); // Let the current rank get the appropriate index in this map. TriCrsMatrix indicator( mpiMap, numOwned + numPresent ); + // ones is a vector of size numOwned of 1s, a vector of size numOther of 1s, or a vector with a single entry of 1, whichever is biggest + // this is just a little trick to ensure you have a long enough vector or 1 to enter into the matrix std::vector< TriScalarInt > const ones( std::max( { numOwned, numPresent, std::size_t( 1 ) } ), 1 ); + indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numOwned ), ones.data(), ownedGlbIdcs.data() ); indicator.insertGlobalValues( TriGlbIdx( curRank.get() ), TriLocIdx( numPresent ), ones.data(), presentGlbIdcs.data() ); indicator.fillComplete( ownedMap, mpiMap ); // The `ownership` matrix is a diagonal square matrix. - // Each size being the number of geometrical entities. + // Each size being the number of geometrical entities (so same size as adjacency). // The value of the diagonal term will be the owning rank. - // By means of matrices multiplications, it will be possible to exchange the ownerships information across the ranks. + // By means of matrix multiplications, it will be possible to exchange the ownerships information across the ranks. // TODO Could we use an Epetra_Vector as a diagonal matrix? std::vector< TriScalarInt > myRank( 1, curRank.get() ); TriCrsMatrix ownership( ownedMap, 1 ); @@ -705,6 +843,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & ownership.fillComplete( ownedMap, ownedMap ); Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ownership.mat", ownership ); + // Log some info on these last 2 matrices created for debugging if( curRank == 0_mpi ) { GEOS_LOG_RANK( "indicator.NumGlobalCols() = " << indicator.getGlobalNumCols() ); @@ -716,24 +855,27 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // The `ghostingFootprint` matrix is rectangular, // the number of columns being the number of MPI ranks, - // the number of rows being the number of nodes in the mesh graph. + // the number of rows being the number of nodes in the mesh graph, ie the total number of mesh geometric quantities + // (So it is the same size as the transpose of the domain indicator matrix) // - // For any MPI rank (ie column), a non-zero entry means + // For any MPI rank (ie column), a non-zero entry in a row means // that the corresponding geometrical entry has to be eventually present onto the rank // for the ghosting to be effective. // // From `ghostingFootprint` we can extract where the current rank has to send any owned graph node. + // Computed using the custom multiply function defined above, + // which does the equivalent of graph tranversal by repeated multiplications of upward and upward^T TriCrsMatrix ghostingFootprint( multiply( commSize, indicator, upward ) ); ghostingFootprint.resumeFill(); - ghostingFootprint.setAllToScalar( 1. ); + ghostingFootprint.setAllToScalar( 1. ); // We put `1` everywhere there's a non-zero entry, so we'll be able to compose with the `ownership` matrix. ghostingFootprint.fillComplete( mpiMap, ownedMap ); Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ghostingFootprint.mat", ghostingFootprint ); - // We put `1` everywhere there's a non-zero entry, so we'll be able to compose with the `ownership` matrix. // FIXME TODO WARNING From `ghostingFootprint` extract where I have to send // Then, perform the ghostingFootprint * ownership matrix multiplication, // so I get to know from whom I'll receive. + // just some output for sanity checks if( curRank == 0_mpi ) { GEOS_LOG_RANK( "ghosted->NumGlobalCols() = " << ghostingFootprint.getGlobalNumCols() ); @@ -741,21 +883,28 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } // The `ghostExchange` matrix is rectangular, - // the number of columns being the number of nodes in the mesh graph, + // the number of columns being the number of nodes in the mesh graph, ie the total number of mesh geometric quantities // the number of rows being the number of MPI ranks. + // ghostExchange (num_ranks x num_entities) = ghostingFootprint^T (num_ranks x num_entities) * ownership (num_entities x num_entities, diagonal) + // recall that ghostExchange has all the nonzeros set to 1 + // diagonal matrix just multiplies each col of ghostingFootprint^T (row of ghostingFootprint) by the corresponding diagonal entry + // So each column of ghostExchange will have nonzeros which are all the same value, with the value being the rank owning that entity + // Thus the nonzeros in row i tell us the ranks which own the entities "adjacent" to the entities on rank i // // As the result of the multiplication between the `ghostingFootprint` matrix and the `ownership` matrix, // for each row owned (ie at the current MPI rank index), - // the value of the `ghostExchange` matrix term will provide the actual owning rank for all the . + // the value of the `ghostExchange` matrix term will provide the actual owning rank for all the needed geometric entities. // // From `ghostExchange` we can extract which other rank will send to the current rank any graph node. - TriCrsMatrix ghostExchange( mpiMap, 10000 ); // TODO having `0` there should be working! + TriCrsMatrix ghostExchange( mpiMap, 10000 ); // TODO having `0` there should be working! We need to get a better estimate of number of entries Tpetra::MatrixMatrix::Multiply( ghostingFootprint, true, ownership, false, ghostExchange, false ); ghostExchange.fillComplete( ownedMap, mpiMap ); // TODO Do I have to work with `ghostingFootprint` if I already have `ghostExchange` which may convey more information? + // As we shall see, below we go back to working with ghostingFootprint // TODO Maybe because of the ownership of the ranks: one is also the "scaled" transposed of the other. + // debug output if( curRank == 0_mpi ) { GEOS_LOG_RANK( "ghostExchange->NumGlobalCols() = " << ghostExchange.getGlobalNumCols() ); @@ -763,20 +912,32 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ghostInfo.mat", ghostExchange ); + // Now, for each of the entities owned by this current rank, we collect all the other ranks to which this entity is sent + + // GhostSend defined in BuildPods.hpp, + // struct of maps for each (node, edge, face, cell) giving the set of ranks each entity (given by node, edge, face, cell global index) must be sent to + GhostSend send; + + // temp data structures used in process std::size_t extracted = 0; Teuchos::Array< TriScalarInt > extractedValues; Teuchos::Array< TriGlbIdx > extractedIndices; - GhostSend send; + // Loop over the owned indiced in global matrix (owned geometric entities) for( TriGlbIdx const & index: ownedGlbIdcs ) { + // get the number of ranks which needed this entity and resize our temp data storage std::size_t const length = ghostingFootprint.getNumEntriesInGlobalRow( index ); - extractedValues.resize( length ); + extractedValues.resize( length ); // note we never actually use the values, just needed the indices where the nonzeros were extractedIndices.resize( length ); + // copy data into temp https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1RowMatrix.html#a5fd9651c8e5d1cc7dead6cadac342978 ghostingFootprint.getGlobalRowCopy( index, extractedIndices(), extractedValues(), extracted ); GEOS_ASSERT_EQ( extracted, length ); + // neighbors will be the set of mpi ranks which need this entity std::set< MpiRank > neighbors; + + // the ranks are the columns of ghostingFootprint, we just extracted the nonzeros for( std::size_t i = 0; i < extracted; ++i ) { MpiRank const rank( extractedIndices[i] ); @@ -786,11 +947,14 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } } + // if this entity is not needed by anyone else, great, we're done if( std::empty( neighbors ) ) { continue; } + // if there ranks which need this entity, we convert the global matrix index back to the (node, edge, face cell) global index + // and add to the GhostSend struct switch( convert.getGeometricalType( index ) ) { case Geom::NODE: @@ -818,15 +982,24 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & GEOS_ERROR( "Internal error" ); } } - } + } // end loop over owned entities + + // Now we can look at the dual problem, namely for this current rank, what data is needed from other ranks + // Now we also have to pay attention to what non-owned data was already present on the rank + // Note that we are back to working with ghostExchange, not ghostingFootprint + // recall that the nonzeros in each row of this matrix told us which other ranks' data was needed for the current row rank std::size_t const numNeededIndices = ghostExchange.getNumEntriesInGlobalRow( TriGlbIdx( curRank.get() ) ); + + // copy into same temp vars as above extractedValues.resize( numNeededIndices ); extractedIndices.resize( numNeededIndices ); ghostExchange.getGlobalRowCopy( TriGlbIdx( curRank.get() ), extractedIndices(), extractedValues(), extracted ); GEOS_ASSERT_EQ( extracted, numNeededIndices ); + // create a set containing the indices needed, i.e. the geometric entities belonging to other ranks (we use a set so we can later use set operations) std::set< TriGlbIdx > const allNeededIndices( std::cbegin( extractedIndices ), std::cend( extractedIndices ) ); + std::set< TriGlbIdx > receivedIndices; // The graph nodes that my neighbors will send me. { std::set< TriGlbIdx > const tmp( std::cbegin( ownedGlbIdcs ), std::cend( ownedGlbIdcs ) ); @@ -834,28 +1007,42 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & std::cbegin( tmp ), std::cend( tmp ), std::inserter( receivedIndices, std::end( receivedIndices ) ) ); } + // of course, some of these indices were already communicated, they were the `present` data std::vector< TriGlbIdx > notPresentIndices; // The graphs nodes that are nor owned neither present by/on the current rank. std::set_difference( std::cbegin( receivedIndices ), std::cend( receivedIndices ), std::cbegin( presentGlbIdcs ), std::cend( presentGlbIdcs ), std::back_inserter( notPresentIndices ) ); + + // a bit below we will see that we need to actually send the positions of the ghosted nodes (everything else is just indices so far) + // so here we go through the notPresentIndices and make note of which are nodes (as opposed to edges, faces, cells) std::vector< TriGlbIdx > notPresentNodes; std::copy_if( std::cbegin( notPresentIndices ), std::cend( notPresentIndices ), std::back_inserter( notPresentNodes ), [&]( TriGlbIdx const & i ) { return convert.getGeometricalType( i ) == Geom::NODE; } ); - GEOS_ASSERT_EQ( std::size( allNeededIndices ), numNeededIndices ); + + // Finally we can construct our received entity information + // GhostRecv defined in BuildPods.hpp, + // struct of maps for each (node, edge, face, cell) giving the rank each entity (given by node, edge, face, cell global index) is coming from GhostRecv recv; for( std::size_t i = 0; i < extracted; ++i ) { + // matrix index of required entity TriGlbIdx const & index = extractedIndices[i]; + + // Note this is not notPresentIndices, as information which is defined on geometric entities + // which happen to already be present will still need to be communicated in the future if( receivedIndices.find( index ) == std::cend( receivedIndices ) ) // TODO make a map `receivedIndices -> mpi rank` { continue; } - MpiRank const sender( extractedValues[i] ); + // get the owning rank of the required entity + MpiRank const sender( extractedValues[i] ); // finally making use of the values from ghostExchange + + // convert matrix index to geometry index and populate GhostRecv switch( convert.getGeometricalType( index ) ) { case Geom::NODE: @@ -885,13 +1072,14 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } } + // debug output // if( curRank == 1_mpi or curRank == 2_mpi ) // { // GEOS_LOG_RANK( "recv.edges = " << json( recv.edges ) ); // GEOS_LOG_RANK( "send.edges = " << json( send.edges ) ); // } - // At this point, each rank knows what it has to send to and what it is going to receive from the other ranks. + // At this point, each rank knows what it has to send and to whom and what it is going to receive from the other ranks. // // The remaining action is about // - retrieving the additional graph information @@ -900,10 +1088,13 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // knowing the index of the nodes is not enough. // // In order to do that, we build the `missingIndicator` matrix, which is rectangular: - // - The number of columns is the number of graph nodes in the mesh graph. + // - The number of columns is the number of graph nodes in the mesh graph, ie the total number of geometrical entities in the mesh. // - The number of rows is the total number of graph nodes that are missing on ranks. + // For me this was confusing, as its a little different from what we've done before + // Each rank is missing some quantity of mesh entities, and we are going to MpiAllreduce to get the total across all ranks, and this is the number of rows in the global matrix + // Its going to be incredible sparse, in fact only one nonzero per row marking the index of the entity in the graph // Note that the same quantity can be missing on multiple ranks, and that's handled. - // - The non-zero terms equal `1`, meaning that a given quantity (column) is missing on a given rank (row). + // - The non-zero terms equal `1`, meaning that a given quantity (column) is missing on a given rank (row, each rank owns a chunk of rows). // // Combining `missingIndicator` with the adjacency matrix which conveys a lot of connections information, // we'll be able to create the final `missingIndices` matrix, @@ -924,21 +1115,31 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // that actually are physical nodes in the mesh. // - The number of rows is the total number of graph nodes (that actually are physical nodes in the mesh) that are missing on ranks. // - The non-zero terms equal `1`, meaning that a given quantity (column) is missing on a given rank (row). + + // Group together the number of missing mesh entities and the number of missing mesh nodes in particular for this rank std::size_t const numLocMissingCompound[2] = { std::size( notPresentIndices ), std::size( notPresentNodes ) }; + // Now we can communicate to get the sum over missing entities and missing mesh nodes over all the ranks with simple reduction std::size_t numGlbMissingCompound[2] = { 0, 0 }; MpiWrapper::allReduce( numLocMissingCompound, numGlbMissingCompound, 2, MPI_SUM ); + + // break the above back into individual variables now that parallel reduction is done std::size_t const & numLocMissingIndices = numLocMissingCompound[0]; std::size_t const & numLocMissingNodes = numLocMissingCompound[1]; Tpetra::global_size_t const numGlbMissingIndices = numGlbMissingCompound[0]; Tpetra::global_size_t const numGlbMissingNodes = numGlbMissingCompound[1]; - std::size_t const numGlbNodes = gis.nodes.get() + 1; // TODO Use strongly typed integers + std::size_t const numGlbNodes = gis.nodes.get() + 1; // TODO Use strongly typed integers, note gis was the input variable desribing global index offsets std::size_t const numOwnedNodes = std::size( ownedNodesIdcs ); // TODO Use strongly typed integers - auto const missingIndicesMap = make_rcp< TriMap const >( numGlbMissingIndices, numGlbMissingIndices, TriGlbIdx{ 0 }, comm ); + // Now we are going to make new ownership maps for the new matrices describing missing indices/nodes + // Different constructor, just using sizes + // https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1Map.html#a7984262c1e6ab71ad3717fd96ed76052 + auto const missingIndicesMap = make_rcp< TriMap const >( numGlbMissingIndices, numGlbMissingIndices, TriGlbIdx{ 0 }, comm ); // TODO: Should second one be local?? auto const missingNodesMap = make_rcp< TriMap const >( numGlbMissingNodes, numLocMissingNodes, TriGlbIdx{ 0 }, comm ); // Following information is needed to compute the global row. + // https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1Map.html#a0124c1b96f046c824123fb0ff5a9c978 + // getting the index in the global matrix corresponding to local index 0 on current rank TriGlbIdx const missingIndicesOffset = missingIndicesMap->getGlobalElement( TriLocIdx{ 0 } ); // Let trilinos provide the offset. TriGlbIdx const missingNodesOffset = missingNodesMap->getGlobalElement( TriLocIdx{ 0 } ); // Let trilinos provide the offset. @@ -946,18 +1147,30 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & TriCrsMatrix missingIndicator( missingIndicesMap, 1 ); for( std::size_t i = 0; i < numLocMissingIndices; ++i ) { - missingIndicator.insertGlobalValues( missingIndicesOffset + TriGlbIdx( i ), TriLocIdx( 1 ), ones.data(), ¬PresentIndices[i] ); + // every row corresponds to a missing entity on this rank + // simply insert a 1 into the column of the entity given by notPresentIndices + // note each rank owns a chunk of consecutive rows + // https://docs.trilinos.org/dev/packages/tpetra/doc/html/classTpetra_1_1CrsMatrix.html#a8d4784081c59f27391ad825c4dae5e86 + missingIndicator.insertGlobalValues( missingIndicesOffset + TriGlbIdx( i ), TriLocIdx( 1 ), ones.data(), ¬PresentIndices[i] ); // note we are re-using ones from wayy back, just need a pointer to something with 1 } missingIndicator.fillComplete( ownedMap, missingIndicesMap ); - // `missingIndices` will contain the missing connectivity information. - TriCrsMatrix missingIndices( missingIndicesMap, 1000 ); // TODO having `0` there should be working! + // Now build `missingIndices` which will contain the missing connectivity information. Same size as missingIndicator + // missingIndices (num_global_missing_indices x num_entities) = missingIndicator (num_global_missing_indices x num_entities) * upward (num_entities x num_entities) + // every row of missingIndicator just had a 1 in the column which denotes the missing entity, upward is or adjacency matrix + // Similar to how if we do upward * (col indicator vector) we go up the topological chain (1 in edge slot yields faces containing edge) + // when we do (indicator row vector) * upward we get the row of upward corresponding to the indicated column, i.e. the indicated mesh entitiy and everything it depends on + // so if the indicator was on a face, you would get back the edges it is composed of as these are the nonzeros in the result vector + // So the full matrix just has this for every row, where each row is one of the missing entities for one of the ranks + // Complete tangent (sort of) - these pictures always help me with the visualizations of matrix omultiplcations that are very helpful here: https://dzone.com/articles/visualizing-matrix + TriCrsMatrix missingIndices( missingIndicesMap, 1000 ); // TODO having `0` there should be working! (epetra could dynamically resize) This is the same TODO as before, need a way to tell tpetra how many nonzeros Tpetra::MatrixMatrix::Multiply( missingIndicator, false, upward, false, missingIndices, false ); missingIndices.fillComplete( ownedMap, missingIndicesMap ); + // Now we can repeat the process but only consider the nodes, as discussed before // Indicator matrix only for the nodes. // Note that it should be an integer matrix full of ones, - // but it's a double matrix because it's going to be multiplied a double matrix. + // but it's a double matrix because it's going to be multiplied by a double matrix (nodal coordinates). TriDblCrsMatrix missingNodesIndicator( missingNodesMap, 1 ); for( std::size_t i = 0; i < numLocMissingNodes; ++i ) { @@ -972,6 +1185,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // - Its number of columns is 3: the x, y, and z coordinates of the nodes. TriDblCrsMatrix nodePositions( ownedNodesMap, 3 ); std::vector< TriGlbIdx > const zot{ TriGlbIdx( 0 ), TriGlbIdx( 1 ), TriGlbIdx( 2 ) }; // zot: zero, one, two. + + // each rank fills the nodePositions for the nodes it owns for( auto const & [ngi, pos]: owned.n2pos ) { nodePositions.insertGlobalValues( convert.fromNodeGlbIdx( ngi ), TriLocIdx( 3 ), pos.data(), zot.data() ); @@ -980,22 +1195,36 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & nodePositions.fillComplete( threeMap, ownedNodesMap ); // `missingNodePos` will contain the missing node positions. - TriDblCrsMatrix missingNodePos( missingNodesMap, 1000 ); // TODO having `0` there should be working! + // Analogous to missingIndices, each row is a node missing from one of the ranks, and the cols are the position of that node + TriDblCrsMatrix missingNodePos( missingNodesMap, 1000 ); // TODO having `0` there should be working! Same TODO for sizing Tpetra::MatrixMatrix::Multiply( missingNodesIndicator, false, nodePositions, false, missingNodePos, false ); missingNodePos.fillComplete( threeMap, missingNodesMap ); + // Okie dokie, with all that we can finally construct the last output, which is the MeshGraph corresponding to ghosted info + // (recall we already had MeshGraphs for the owned and present info) + // (and recall a MeshGraph is just a struct of maps which take you from node/edge/face/cell global index to its data, + // ex: face global id maps to a vector of edges, each of which are encoded by EdgeInfo (so it has id, start node)) + // Notice that this information is exactly what is in missingIndices and missingNodePos! + // Moreoever, we will be able to get back 'all' the information by decoding the entries of missingIndices! MeshGraph ghosts; Teuchos::Array< double > extractedNodePos( 3 ); + + // Loop over the missing mesh entities for( std::size_t i = 0; i < numLocMissingIndices; ++i ) { + // This is the missing entity that will be ghosted, this will be the key in one of the ghosted maps TriGlbIdx const index = notPresentIndices[i]; Geom const geometricalType = convert.getGeometricalType( index ); + // get the number of entities the missing one depends on (downward, c -> f -> e -> n) std::size_t const length = missingIndices.getNumEntriesInGlobalRow( missingIndicesOffset + TriGlbIdx( i ) ); + + // reuse same temp vars from before to store the row data extractedValues.resize( length ); extractedIndices.resize( length ); missingIndices.getGlobalRowCopy( missingIndicesOffset + TriGlbIdx( i ), extractedIndices(), extractedValues(), extracted ); GEOS_ASSERT_EQ( extracted, length ); + if( geometricalType == Geom::NODE ) { // The case of nodes is a bit different from the other cases, @@ -1016,32 +1245,40 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & continue; } - auto const cit = std::find( std::cbegin( extractedIndices ), std::cend( extractedIndices ), index ); + // Now we are either edge, face, or cell + + // Find the index of the current missing entity in the data from the row of the matrix + auto const cit = std::find( std::cbegin( extractedIndices ), std::cend( extractedIndices ), index ); // I kind of think it should always be the last one, since upward is (lower) triangular + // Get the diagonal value std::size_t const numGeomQuantitiesIdx = intConv< std::size_t >( std::distance( std::cbegin( extractedIndices ), cit ) ); TriScalarInt const & numGeomQuantities = extractedValues[numGeomQuantitiesIdx]; - GEOS_ASSERT_EQ( extracted, intConv< std::size_t >( numGeomQuantities + 1 ) ); + GEOS_ASSERT_EQ( extracted, intConv< std::size_t >( numGeomQuantities + 1 ) ); // extracted is the output from Tilinos telling you how many entries were pulled out when we copy row switch( geometricalType ) { case Geom::EDGE: { TriScalarInt const & numNodes = numGeomQuantities; // Alias - GEOS_ASSERT_EQ( numNodes, 2 ); + GEOS_ASSERT_EQ( numNodes, 2 ); // an edge always has 2 nodes std::array< NodeGlbIdx, 2 > order{}; + // loop over the entries extracted from the row of the matrix for( std::size_t ii = 0; ii < extracted; ++ii ) { + // skip diagonal if( ii == numGeomQuantitiesIdx ) { continue; } + // use `order` to store [start node index, end node index] NodeGlbIdx const ngi = convert.toNodeGlbIdx( extractedIndices[ii] ); TriScalarInt const ord = extractedValues[ii] - 1; GEOS_ASSERT( ord == 0 or ord == 1 ); order[ord] = ngi; } GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numNodes ) ); + // populate this info into ghosts EdgeGlbIdx const egi = convert.toEdgeGlbIdx( index ); std::tuple< NodeGlbIdx, NodeGlbIdx > & tmp = ghosts.e2n[egi]; std::get< 0 >( tmp ) = order[0]; @@ -1052,19 +1289,27 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & { TriScalarInt const & numEdges = numGeomQuantities; // Alias std::map< integer, EdgeInfo > order; + + // loop over the entries extracted from the row of the matrix for( std::size_t ii = 0; ii < extracted; ++ii ) { + // skip diagonal if( ii == numGeomQuantitiesIdx ) { continue; } + // get the edge global index and decode the matrix entry to get each edge's start node and the edges index in the face EdgeGlbIdx const egi = convert.toEdgeGlbIdx( extractedIndices[ii] ); + // array is [start node, index within face] std::array< std::size_t, 2 > const decoded = decode< 2 >( numEdges, extractedValues[ii] - 1 ); + // `order` holds all the edge infos in the order the edges appeared in the original face order[decoded[1]] = { egi, intConv< std::uint8_t >( decoded[0] ) }; GEOS_ASSERT( decoded[0] == 0 or decoded[0] == 1 ); } GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numEdges ) ); + + // use face global index and `order` to populate ghosts FaceGlbIdx const fgi = convert.toFaceGlbIdx( index ); std::vector< EdgeInfo > & tmp = ghosts.f2e[fgi]; tmp.resize( numEdges ); @@ -1079,18 +1324,26 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & { TriScalarInt const & numFaces = numGeomQuantities; // Alias // TODO This should receive the cell type instead. std::map< integer, FaceInfo > order; + + // loop over the entries extracted from the row of the matrix for( std::size_t ii = 0; ii < extracted; ++ii ) { + // skip diagonal if( ii == numGeomQuantitiesIdx ) { continue; } + // get the face global index and decode the entries to get isFlipped, start index, and index of face within cell FaceGlbIdx const fgi = convert.toFaceGlbIdx( extractedIndices[ii] ); + // array is [isFlipped, start index, and index of face within cell] std::array< std::size_t, 3 > const decoded = decode< 3 >( numFaces, extractedValues[ii] - 1 ); + // `order` holds the face info in the order the faces appeared in the original cell order[decoded[2]] = { fgi, intConv< bool >( decoded[0] ), intConv< std::uint8_t >( decoded[1] ) }; } GEOS_ASSERT_EQ( std::size( order ), intConv< std::size_t >( numFaces ) ); + + // use cell global index and `order` to populate ghosts CellGlbIdx const cgi = convert.toCellGlbIdx( index ); std::vector< FaceInfo > & tmp = ghosts.c2f[cgi]; tmp.resize( numFaces ); @@ -1102,7 +1355,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } default: { - GEOS_ERROR( "Internal error." ); + GEOS_ERROR( "Internal error in performGhosting. Could not recognize geometric type of mesh entity" ); } } } @@ -1127,6 +1380,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // GEOS_LOG_RANK( "ghosts_n2ps = " << json( ghosts.n2pos ) ); // } + // Now we are done!!! return { ghosts, recv, send }; } @@ -1134,6 +1388,11 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, std::set< MpiRank > const & neighbors, MeshMappingImpl & meshMappings ) { + + // First step in ghosting is to have each rank create the global IDs for each entry appearing on it + // buckets is a group of maps from sets of mpi ranks to shared (nodes, edges, faces) + // offsets has the same keys but the values are now the global index for the first entry in the bucket + // the rest of the items in a bucket are numered sequentially following the value of offset auto const [buckets, offsets] = doTheNewGlobalNumbering( mesh, neighbors ); // Now we exchange the data with our neighbors. @@ -1141,10 +1400,20 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, GEOS_LOG_RANK( "offsets on rank " << curRank << " -> " << json( offsets ) ); + // Now we grab the biggest global indices for each (nodes, edges, faces, cells) as this will help build the adjacency matrix + // gatherOffset does an allReduce of max indices on each rank + // note we use the extra entry in the offsets for the next rank which specifies where it starts to get the max index on this rank + // so we dont have to do any extra counting or anything in the current rank buckets MpiRank const nextRank = curRank + 1_mpi; MaxGlbIdcs const matrixOffsets = gatherOffset( mesh, offsets.edges.at( { nextRank } ) - 1_egi, offsets.faces.at( { nextRank } ) - 1_fgi ); std::cout << "matrixOffsets on rank " << curRank << " -> " << json( matrixOffsets ) << std::endl; + // Now we build to data to describe the mesh data on the current rank. + // owned and present are struct of maps which map: + // cell global index to vector of entries of type (face global index, extra stuff defining orientation of face) for each of the adjacent faces + // face global index to vector of entries of type (edge global index plus orientation) for each of the adjacent edges + // edge global index to tuple of node global indices + // node global index to 3d array of position auto const [owned, present] = buildMeshGraph( mesh, buckets, offsets, curRank ); // TODO change into buildOwnedMeshGraph? // if( curRank == 1_mpi ) // { @@ -1153,8 +1422,17 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, // } // MpiWrapper::barrier(); + // Next do the actual ghosting - i.e. figure out the information each rank must send to and receive from other ranks + // This is where we use parallel linear algebra package (right now, Trilinos) to build an adjacency matrix for the entire mesh (treated as a graph) + // By doing some clever matrix products we can figure out which ranks need what automagically + // `ghosts` is also a MeshGraph like owned and present, but described the geometric entities which are ghosted onto the rank from another owner + // recv and send are of type GhostRecv and GhostSend (in BuildPods.hpp) and describe what (nodes, edges, faces, cells) each rank needs to receive from and send to others auto const [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); + // Finally, we use everything we have to populate the mappings that interface with the rest of GEOS + // Note that we have already done the ghosting, so these mappings are set once and for all + // Unlike previous implementation, where we populated these based on the current rank info, and then tried to fix any inconsistencies with ghosting + // output is the populated meshMappings, which contains node manager, edge manager, etc. as well as the ghost send/recv buildPods( owned, present, ghosts, recv, send, meshMappings ); } diff --git a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp index 62d6c34e61d..e9fc77c6cdb 100644 --- a/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp +++ b/src/coreComponents/mesh/generators/NewGlobalNumbering.cpp @@ -31,583 +31,637 @@ namespace geos::ghosting { -struct Exchange -{ - std::set< NodeGlbIdx > nodes; - std::set< Edge > edges; - std::set< Face > faces; -}; - -void to_json( json & j, - const Exchange & v ) -{ - j = json{ { "nodes", v.nodes }, - { "edges", v.edges }, - { "faces", v.faces } }; -} + struct Exchange + { + std::set nodes; + std::set edges; + std::set faces; + }; -void from_json( const json & j, - Exchange & v ) -{ - v.nodes = j.at( "nodes" ).get< std::set< NodeGlbIdx > >(); - v.edges = j.at( "edges" ).get< std::set< Edge > >(); - v.faces = j.at( "faces" ).get< std::set< Face > >(); -} + void to_json(json &j, + const Exchange &v) + { + j = json{{"nodes", v.nodes}, + {"edges", v.edges}, + {"faces", v.faces}}; + } -array1d< std::uint8_t > convertExchange( Exchange const & exchange ) -{ - std::vector< std::uint8_t > const tmp = json::to_cbor( json( exchange ) ); - array1d< std::uint8_t > result; - result.reserve( tmp.size() ); - for( std::uint8_t const & t: tmp ) + void from_json(const json &j, + Exchange &v) { - result.emplace_back( t ); + v.nodes = j.at("nodes").get>(); + v.edges = j.at("edges").get>(); + v.faces = j.at("faces").get>(); } - return result; -} -Exchange convertExchange( array1d< std::uint8_t > const & input ) -{ - std::vector< std::uint8_t > const tmp( std::cbegin( input ), std::cend( input ) ); - return json::from_cbor( tmp, false ).template get< Exchange >(); -} - -/** - * @brief Extract the cells at the boundary of the mesh. - * @param mesh The vtk mesh. - * @return The vtk cell ids. - */ -std::set< vtkIdType > extractBoundaryCells( vtkSmartPointer< vtkDataSet > mesh ) -{ - // TODO Better handle the boundary information, forgetting about the 3d cells and simply handling the outside shell. - auto f = vtkDataSetSurfaceFilter::New(); - f->PassThroughCellIdsOn(); - f->PassThroughPointIdsOff(); - f->FastModeOff(); - - string const originalCellsKey = "ORIGINAL_CELLS"; - f->SetOriginalCellIdsName( originalCellsKey.c_str() ); - auto boundaryMesh = vtkPolyData::New(); - f->UnstructuredGridExecute( mesh, boundaryMesh ); - vtkIdTypeArray const * originalCells = vtkIdTypeArray::FastDownCast( boundaryMesh->GetCellData()->GetArray( originalCellsKey.c_str() ) ); - - std::set< vtkIdType > boundaryCellIdxs; - for( auto i = 0; i < originalCells->GetNumberOfTuples(); ++i ) + array1d convertExchange(Exchange const &exchange) { - boundaryCellIdxs.insert( originalCells->GetValue( i ) ); + std::vector const tmp = json::to_cbor(json(exchange)); + array1d result; + result.reserve(tmp.size()); + for (std::uint8_t const &t : tmp) + { + result.emplace_back(t); + } + return result; } - return boundaryCellIdxs; -} + Exchange convertExchange(array1d const &input) + { + std::vector const tmp(std::cbegin(input), std::cend(input)); + return json::from_cbor(tmp, false).template get(); + } -Face reorderFaceNodes( std::vector< NodeGlbIdx > const & nodes, bool & isFlipped, std::uint8_t & start ) // TODO unit test -{ - std::size_t const n = nodes.size(); + /** + * @brief Extract the cells at the boundary of the mesh. + * @param mesh The vtk mesh. + * @return The vtk cell ids. + */ + std::set extractBoundaryCells(vtkSmartPointer mesh) + { + // TODO Better handle the boundary information, forgetting about the 3d cells and simply handling the outside shell. + auto f = vtkDataSetSurfaceFilter::New(); + f->PassThroughCellIdsOn(); + f->PassThroughPointIdsOff(); + f->FastModeOff(); + + string const originalCellsKey = "ORIGINAL_CELLS"; + f->SetOriginalCellIdsName(originalCellsKey.c_str()); + auto boundaryMesh = vtkPolyData::New(); + f->UnstructuredGridExecute(mesh, boundaryMesh); + vtkIdTypeArray const *originalCells = vtkIdTypeArray::FastDownCast(boundaryMesh->GetCellData()->GetArray(originalCellsKey.c_str())); + + std::set boundaryCellIdxs; + for (auto i = 0; i < originalCells->GetNumberOfTuples(); ++i) + { + boundaryCellIdxs.insert(originalCells->GetValue(i)); + } - std::vector< NodeGlbIdx > f( nodes ); + return boundaryCellIdxs; + } + // Reorder nodes by starting with the smallest node index, and then iterating in the direction of the smaller of its neighbors + Face reorderFaceNodes(std::vector const &nodes, bool &isFlipped, std::uint8_t &start) // TODO unit test + { + std::size_t const n = nodes.size(); - auto const cit = std::min_element( std::cbegin( nodes ), std::cend( nodes ) ); - auto const minIdx = std::distance( std::cbegin( nodes ), cit ); + std::vector f(nodes); - std::size_t const prevIdx = minIdx == 0 ? n - 1 : minIdx - 1; - std::size_t const nextIdx = minIdx == intConv< int >( n ) - 1 ? 0 : minIdx + 1; + // Get the smallest global node Id in the face + auto const cit = std::min_element(std::cbegin(nodes), std::cend(nodes)); + auto const minIdx = std::distance(std::cbegin(nodes), cit); - start = intConv< std::uint8_t >( minIdx ); - isFlipped = nodes[prevIdx] < nodes[nextIdx]; + // get the adjecent node gloabl IDs in he vector (treated as a circular buffer) + std::size_t const prevIdx = minIdx == 0 ? n - 1 : minIdx - 1; + std::size_t const nextIdx = minIdx == intConv(n) - 1 ? 0 : minIdx + 1; - std::size_t const pivot = isFlipped ? n - 1 - minIdx : minIdx; - if( isFlipped ) - { - std::reverse( std::begin( f ), std::end( f ) ); - } - std::rotate( std::begin( f ), std::begin( f ) + pivot, std::end( f ) ); + // we will order by starting with the smallest ID and then iterating in the direction of its smaller neighbor + // this is encoded by isFlipped boolean (true if we iterate "left" from the start) + start = intConv(minIdx); + isFlipped = nodes[prevIdx] < nodes[nextIdx]; - return f; -} + // return the re-ordered nodes vector, isFlipped and start have also been set by this function + std::size_t const pivot = isFlipped ? n - 1 - minIdx : minIdx; + if (isFlipped) + { + std::reverse(std::begin(f), std::end(f)); + } + std::rotate(std::begin(f), std::begin(f) + pivot, std::end(f)); -std::vector< NodeLocIdx > resetFaceNodes( std::vector< NodeLocIdx > const & nodes, - bool const & isFlipped, - std::uint8_t const & start ) -{ - std::vector< NodeLocIdx > result( nodes ); - if( isFlipped ) - { - std::reverse( std::begin( result ), std::end( result ) ); - std::uint8_t const pivot = std::size( result ) - 1 - start; - std::rotate( std::begin( result ), std::begin( result ) + pivot, std::end( result ) ); + return f; } - else - { - std::rotate( std::rbegin( result ), std::rbegin( result ) + start, std::rend( result ) ); - } - return result; -} - -Exchange buildSpecificData( vtkSmartPointer< vtkDataSet > mesh, - std::set< vtkIdType > const & cellIds ) -{ - vtkIdTypeArray * gids = vtkIdTypeArray::FastDownCast( mesh->GetPointData()->GetGlobalIds() ); - Span< vtkIdType > const globalPtIds( (vtkIdType *) gids->GetPointer( 0 ), gids->GetNumberOfTuples() ); - - std::vector< Edge > tmpEdges; - std::vector< Face > tmpFaces; - // Pre-allocation of the temporary vectors. + std::vector resetFaceNodes(std::vector const &nodes, + bool const &isFlipped, + std::uint8_t const &start) { - std::size_t numEdges = 0; - std::size_t numFaces = 0; - for( vtkIdType const & c: cellIds ) + std::vector result(nodes); + if (isFlipped) + { + std::reverse(std::begin(result), std::end(result)); + std::uint8_t const pivot = std::size(result) - 1 - start; + std::rotate(std::begin(result), std::begin(result) + pivot, std::end(result)); + } + else { - vtkCell * cell = mesh->GetCell( c ); - numEdges += cell->GetNumberOfEdges(); - numFaces += cell->GetNumberOfFaces(); + std::rotate(std::rbegin(result), std::rbegin(result) + start, std::rend(result)); } - tmpEdges.reserve( numEdges ); - tmpFaces.reserve( numFaces ); + return result; } - // Filling the temporary. - for( auto c = 0; c < mesh->GetNumberOfCells(); ++c ) + Exchange buildSpecificData(vtkSmartPointer mesh, + std::set const &cellIds) { - vtkCell * cell = mesh->GetCell( c ); + vtkIdTypeArray *gids = vtkIdTypeArray::FastDownCast(mesh->GetPointData()->GetGlobalIds()); + Span const globalPtIds((vtkIdType *)gids->GetPointer(0), gids->GetNumberOfTuples()); - for( auto e = 0; e < cell->GetNumberOfEdges(); ++e ) - { - vtkCell * edge = cell->GetEdge( e ); - vtkIdType const nli0 = edge->GetPointId( 0 ); - vtkIdType const nli1 = edge->GetPointId( 1 ); - vtkIdType const ngi0 = globalPtIds[nli0]; - vtkIdType const ngi1 = globalPtIds[nli1]; - tmpEdges.emplace_back( std::minmax( { NodeGlbIdx{ ngi0 }, NodeGlbIdx{ ngi1 } } ) ); - } + std::vector tmpEdges; + std::vector tmpFaces; - for( auto f = 0; f < cell->GetNumberOfFaces(); ++f ) + // Pre-allocation of the temporary vectors. { - vtkCell * face = cell->GetFace( f ); - vtkIdList * pids = face->GetPointIds(); - std::vector< NodeGlbIdx > nodes( pids->GetNumberOfIds() ); - for( std::size_t i = 0; i < nodes.size(); ++i ) + std::size_t numEdges = 0; + std::size_t numFaces = 0; + for (vtkIdType const &c : cellIds) { - vtkIdType const nli = face->GetPointId( i ); - vtkIdType const ngi = globalPtIds[nli]; - nodes[i] = NodeGlbIdx{ ngi }; + vtkCell *cell = mesh->GetCell(c); + numEdges += cell->GetNumberOfEdges(); + numFaces += cell->GetNumberOfFaces(); } - bool isFlipped; - std::uint8_t start; - tmpFaces.emplace_back( reorderFaceNodes( nodes, isFlipped, start ) ); + tmpEdges.reserve(numEdges); + tmpFaces.reserve(numFaces); } - } - std::set< NodeGlbIdx > nodes; - std::transform( std::cbegin( globalPtIds ), std::cend( globalPtIds ), - std::inserter( nodes, std::end( nodes ) ), []( vtkIdType const & id ) - { return NodeGlbIdx{ id }; } ); - - // Removing the duplicates by copying into a `std::set`. - std::set< Edge > edges{ tmpEdges.cbegin(), tmpEdges.cend() }; // SortedArray requires the input to be sorted already. - std::set< Face > faces{ tmpFaces.cbegin(), tmpFaces.cend() }; + // Filling the temporary. + for (auto c = 0; c < mesh->GetNumberOfCells(); ++c) + { + vtkCell *cell = mesh->GetCell(c); - return { std::move( nodes ), std::move( edges ), std::move( faces ) }; -} + for (auto e = 0; e < cell->GetNumberOfEdges(); ++e) + { + vtkCell *edge = cell->GetEdge(e); + vtkIdType const nli0 = edge->GetPointId(0); + vtkIdType const nli1 = edge->GetPointId(1); + vtkIdType const ngi0 = globalPtIds[nli0]; + vtkIdType const ngi1 = globalPtIds[nli1]; + tmpEdges.emplace_back(std::minmax({NodeGlbIdx{ngi0}, NodeGlbIdx{ngi1}})); + } + for (auto f = 0; f < cell->GetNumberOfFaces(); ++f) + { + vtkCell *face = cell->GetFace(f); + vtkIdList *pids = face->GetPointIds(); + std::vector nodes(pids->GetNumberOfIds()); + for (std::size_t i = 0; i < nodes.size(); ++i) + { + vtkIdType const nli = face->GetPointId(i); + vtkIdType const ngi = globalPtIds[nli]; + nodes[i] = NodeGlbIdx{ngi}; + } + bool isFlipped; + std::uint8_t start; + tmpFaces.emplace_back(reorderFaceNodes(nodes, isFlipped, start)); + } + } -Exchange buildFullData( vtkSmartPointer< vtkDataSet > mesh ) -{ - std::set< vtkIdType > cellIds; - for( vtkIdType i = 0; i < mesh->GetNumberOfCells(); ++i ) - { - cellIds.insert( cellIds.end(), i ); - } + std::set nodes; + std::transform(std::cbegin(globalPtIds), std::cend(globalPtIds), + std::inserter(nodes, std::end(nodes)), [](vtkIdType const &id) + { return NodeGlbIdx{id}; }); - return buildSpecificData( mesh, cellIds ); -} + // Removing the duplicates by copying into a `std::set`. + std::set edges{tmpEdges.cbegin(), tmpEdges.cend()}; // SortedArray requires the input to be sorted already. + std::set faces{tmpFaces.cbegin(), tmpFaces.cend()}; -Exchange buildExchangeData( vtkSmartPointer< vtkDataSet > mesh ) -{ - return buildSpecificData( mesh, extractBoundaryCells( mesh ) ); -} - -/** - * @brief - * @tparam GLB_IDX The global index type of the geometrical quantity considered (typically @c EdgeGlbIdx or @c FaceGlbIdx). - * @param sizes The sizes of the intersection bucket (from the current MPI rank). - * @param offsets The offsets for each intersection bucket (from the MPI_Scan process, i.e. the previous ranks). - * @param curRank Current MPI rank. - * @return The offset buckets, updated with the sizes of the current MPI rank. - */ -template< typename GLB_IDX > -std::map< std::set< MpiRank >, GLB_IDX > updateBucketOffsets( std::map< std::set< MpiRank >, localIndex > const & sizes, - std::map< std::set< MpiRank >, GLB_IDX > const & offsets, - MpiRank curRank ) -{ - std::map< std::set< MpiRank >, GLB_IDX > reducedOffsets; + return {std::move(nodes), std::move(edges), std::move(faces)}; + } - // We only keep the offsets that are still relevant to the current and higher ranks. - // Note that `reducedOffsets` will be used by the _current_ rank too. - // Therefore, we'll send information that may not be useful to higher ranks. - // This is surely acceptable because the information is tiny. - for( auto const & [ranks, offset]: offsets ) + Exchange buildFullData(vtkSmartPointer mesh) { - MpiRank const maxConcerned = *std::max_element( ranks.cbegin(), ranks.cend() ); - // TODO invert -// if( maxConcerned >= curRank ) -// { -// reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); -// } - if( maxConcerned < curRank ) + std::set cellIds; + for (vtkIdType i = 0; i < mesh->GetNumberOfCells(); ++i) { - continue; + cellIds.insert(cellIds.end(), i); } - reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); + + return buildSpecificData(mesh, cellIds); + } + + Exchange buildExchangeData(vtkSmartPointer mesh) + { + return buildSpecificData(mesh, extractBoundaryCells(mesh)); } - // Add the offsets associated to the new size buckets from the current rank. - GLB_IDX nextOffset{ 0 }; - for( auto const & [ranks, size]: sizes ) + /** + * @brief + * @tparam GLB_IDX The global index type of the geometrical quantity considered (typically @c EdgeGlbIdx or @c FaceGlbIdx). + * @param sizes The sizes of the intersection bucket (from the current MPI rank). + * @param offsets The offsets for each intersection bucket (from the MPI_Scan process, i.e. the previous ranks). + * @param curRank Current MPI rank. + * @return The offset buckets, updated with the sizes of the current MPI rank. + */ + template + std::map, GLB_IDX> updateBucketOffsets(std::map, localIndex> const &sizes, + std::map, GLB_IDX> const &offsets, + MpiRank curRank) { - auto const it = reducedOffsets.find( ranks ); - if( it == reducedOffsets.end() ) + std::map, GLB_IDX> reducedOffsets; + + // We only keep the offsets that are still relevant to the current and higher ranks. + // Note that `reducedOffsets` will be used by the _current_ rank too. + // Therefore, we'll send information that may not be useful to higher ranks. + // This is surely acceptable because the information is tiny. + // We do this by looping through the buckets, finding the beggest rank, and then only adding the bucket and its offset if biggest rank is >= current rank + for (auto const &[ranks, offset] : offsets) { - reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, nextOffset ); - nextOffset += GLB_IDX{ size }; + MpiRank const maxConcerned = *std::max_element(ranks.cbegin(), ranks.cend()); + // TODO invert + // if( maxConcerned >= curRank ) + // { + // reducedOffsets.emplace_hint( reducedOffsets.end(), ranks, offset ); + // } + if (maxConcerned < curRank) + { + continue; + } + reducedOffsets.emplace_hint(reducedOffsets.end(), ranks, offset); } - else + + // Add the offsets associated to the new size buckets from the current rank. + GLB_IDX nextOffset{0}; + // Loop through th buckets in either (edges, faces) + for (auto const &[ranks, size] : sizes) { - nextOffset = it->second + GLB_IDX{ size }; // Define the new offset from the last + // See if this bucket was in the previously indexed data + // if so, then we just need to update the nextOffset, which will be used for the next new data + // if the data is new, then we add its offset to the set of indexed data in reducedOffsets + auto const it = reducedOffsets.find(ranks); + if (it == reducedOffsets.end()) + { + reducedOffsets.emplace_hint(reducedOffsets.end(), ranks, nextOffset); + nextOffset += GLB_IDX{size}; + } + else + { + nextOffset = it->second + GLB_IDX{size}; // Define the new offset from the last + } } - } - // Add an extra entry based for the following rank - reducedOffsets.emplace_hint( reducedOffsets.end(), std::set< MpiRank >{ curRank + 1_mpi }, nextOffset ); + // Add an extra entry based for the following rank + // There are edge cases where a rank cannot start counting without this + reducedOffsets.emplace_hint(reducedOffsets.end(), std::set{curRank + 1_mpi}, nextOffset); - return reducedOffsets; -} + return reducedOffsets; + } -// TODO Duplicated -std::map< MpiRank, Exchange > exchange( int commId, - std::vector< NeighborCommunicator > & neighbors, - Exchange const & data ) -{ - MPI_iCommData commData( commId ); - integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); - commData.resize( numNeighbors ); + // TODO Duplicated + std::map exchange(int commId, + std::vector &neighbors, + Exchange const &data) + { + MPI_iCommData commData(commId); + integer const numNeighbors = LvArray::integerConversion(neighbors.size()); + commData.resize(numNeighbors); + array1d const cv = convertExchange(data); - array1d< std::uint8_t > const cv = convertExchange( data ); + for (integer i = 0; i < numNeighbors; ++i) + { + neighbors[i].mpiISendReceiveSizes(cv, + commData.mpiSendBufferSizeRequest(i), + commData.mpiRecvBufferSizeRequest(i), + commId, + MPI_COMM_GEOSX); + } - for( integer i = 0; i < numNeighbors; ++i ) - { - neighbors[i].mpiISendReceiveSizes( cv, - commData.mpiSendBufferSizeRequest( i ), - commData.mpiRecvBufferSizeRequest( i ), - commId, - MPI_COMM_GEOSX ); - } + MpiWrapper::waitAll(numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus()); + MpiWrapper::waitAll(numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus()); - MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferSizeRequest(), commData.mpiSendBufferSizeStatus() ); - MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferSizeRequest(), commData.mpiRecvBufferSizeStatus() ); + array1d> rawExchanged(neighbors.size()); - array1d< array1d< std::uint8_t > > rawExchanged( neighbors.size() ); + for (integer i = 0; i < numNeighbors; ++i) + { + neighbors[i].mpiISendReceiveData(cv, + commData.mpiSendBufferRequest(i), + rawExchanged[i], + commData.mpiRecvBufferRequest(i), + commId, + MPI_COMM_GEOSX); + } + MpiWrapper::waitAll(numNeighbors, commData.mpiSendBufferRequest(), commData.mpiSendBufferStatus()); + MpiWrapper::waitAll(numNeighbors, commData.mpiRecvBufferRequest(), commData.mpiRecvBufferStatus()); - for( integer i = 0; i < numNeighbors; ++i ) - { - neighbors[i].mpiISendReceiveData( cv, - commData.mpiSendBufferRequest( i ), - rawExchanged[i], - commData.mpiRecvBufferRequest( i ), - commId, - MPI_COMM_GEOSX ); + std::map output; + for (auto i = 0; i < numNeighbors; ++i) + { + output[MpiRank{neighbors[i].neighborRank()}] = convertExchange(rawExchanged[i]); + } + return output; } - MpiWrapper::waitAll( numNeighbors, commData.mpiSendBufferRequest(), commData.mpiSendBufferStatus() ); - MpiWrapper::waitAll( numNeighbors, commData.mpiRecvBufferRequest(), commData.mpiRecvBufferStatus() ); - std::map< MpiRank, Exchange > output; - for( auto i = 0; i < numNeighbors; ++i ) - { - output[MpiRank{ neighbors[i].neighborRank() }] = convertExchange( rawExchanged[i] ); - } - return output; -} - -/** - * @brief - * @tparam T Typically NodeGloIdx or a container of NodeGlbIdx (for edges and faces) - * @param exchanged - * @param curRank - * @param neighbors - * @return - */ -template< typename T > -std::map< std::set< MpiRank >, std::set< T > > -buildIntersectionBuckets( std::map< MpiRank, std::set< T > const & > const & exchanged, - MpiRank curRank, - std::set< MpiRank > const & neighbors ) -{ - std::map< T, std::set< MpiRank > > counts; // TODO Use better intersection algorithms? - // We "register" all the edges of the current rank: they are the only one we're interested in. - for( T const & node: exchanged.at( curRank ) ) + /** + * @brief + * @tparam T Typically NodeGloIdx or a container of NodeGlbIdx (for edges and faces) + * @param exchanged + * @param curRank + * @param neighbors + * @return + */ + template + std::map, std::set> + buildIntersectionBuckets(std::map const &> const &exchanged, + MpiRank curRank, + std::set const &neighbors) { - counts.emplace_hint( counts.end(), node, std::set< MpiRank >{ curRank } ); - } + // counts - which ranks use each (node, edge, face) + std::map> counts; // TODO Use better intersection algorithms? - // We now loop on the neighbor edges. - // If a neighbor has an edge in common with the current rank, they we store it. - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. - { - for( T const & node: exchanged.at( neighborRank ) ) + // We "register" all the edges of the current rank: they are the only one we're interested in. + // i.e. now exchanged is just one of (nodes, edges, faces), we add keys for each of the (nodes, edges, faces) + // to counts and say it is used by the current rank + for (T const &node : exchanged.at(curRank)) { - auto it = counts.find( node ); - if( it != counts.cend() ) // TODO Extract `counts.cend()` out of the loop. + counts.emplace_hint(counts.end(), node, std::set{curRank}); + } + + // We now loop on the neighbor edges. + // If a neighbor has an edge in common with the current rank, they we store it. + for (MpiRank const &neighborRank : neighbors) // This does not include the current rank. + { + // for each of the (nodes, edges, faces) on the boundary of the other ranks, + // add that that entry is also used by the neighbor rank + for (T const &node : exchanged.at(neighborRank)) { - it->second.insert( neighborRank ); + auto it = counts.find(node); + if (it != counts.cend()) // TODO Extract `counts.cend()` out of the loop. + { + it->second.insert(neighborRank); + } } } - } - std::map< std::set< MpiRank >, std::set< T > > nodeBuckets; - for( auto const & [node, ranks]: counts ) - { - if( ranks.find( curRank ) != ranks.cend() ) + // Counts is essentially the inverse relationship to what we want + // now invert so that we have keys as set of MPI ranks sharing and value as the (node, edge, face) + std::map, std::set> nodeBuckets; + for (auto const &[node, ranks] : counts) { - nodeBuckets[ranks].insert( node ); + if (ranks.find(curRank) != ranks.cend()) + { + nodeBuckets[ranks].insert(node); + } } + return nodeBuckets; } - return nodeBuckets; -} - -/** - * @brief Compute the intersection between for the ranks based on the information @p exchanged. - * @param exchanged Geometrical information provided by the ranks (including the current rank). - * @param curRank The current MPI rank. - * @param neighbors excluding current rank - * @return The intersection buckets. - */ -Buckets buildIntersectionBuckets( std::map< MpiRank, Exchange > const & exchanged, - MpiRank curRank, - std::set< MpiRank > const & neighbors ) -{ - std::map< MpiRank, std::set< NodeGlbIdx > const & > nodeInfo; - for( auto const & [rank, exchange]: exchanged ) - { - nodeInfo.emplace( rank, exchange.nodes ); - } - std::map< std::set< MpiRank >, std::set< NodeGlbIdx > > const nodeBuckets = buildIntersectionBuckets( nodeInfo, curRank, neighbors ); - std::map< MpiRank, std::set< Edge > const & > edgeInfo; - for( auto const & [rank, exchange]: exchanged ) + /** + * @brief Compute the intersection between for the ranks based on the information @p exchanged. + * @param exchanged Geometrical information provided by the ranks (including the current rank). + * @param curRank The current MPI rank. + * @param neighbors excluding current rank + * @return The intersection buckets. + */ + Buckets buildIntersectionBuckets(std::map const &exchanged, + MpiRank curRank, + std::set const &neighbors) { - edgeInfo.emplace( rank, exchange.edges ); - } - std::map< std::set< MpiRank >, std::set< Edge > > const edgeBuckets = buildIntersectionBuckets( edgeInfo, curRank, neighbors ); + // create storage for the set of nodes owned by each rank, then fill using exchange info + // this will be passed to function which computes intersections + std::map const &> nodeInfo; + for (auto const &[rank, exchange] : exchanged) + { + nodeInfo.emplace(rank, exchange.nodes); + } + std::map, std::set> const nodeBuckets = buildIntersectionBuckets(nodeInfo, curRank, neighbors); - // For faces, the algorithm can be a tad simpler because faces can be shared by at most 2 ranks. - std::map< std::set< MpiRank >, std::set< Face > > faceBuckets; - std::set< Face > curFaces = exchanged.at( curRank ).faces; - for( MpiRank const & neighborRank: neighbors ) // This does not include the current rank. - { - for( Face const & face: exchanged.at( neighborRank ).faces ) + // Repeat the process for the edges + std::map const &> edgeInfo; + for (auto const &[rank, exchange] : exchanged) { - auto it = curFaces.find( face ); - if( it != curFaces.cend() ) + edgeInfo.emplace(rank, exchange.edges); + } + std::map, std::set> const edgeBuckets = buildIntersectionBuckets(edgeInfo, curRank, neighbors); + + // For faces, the algorithm can be a tad simpler because faces can be shared by at most 2 ranks. + std::map, std::set> faceBuckets; + std::set curFaces = exchanged.at(curRank).faces; + for (MpiRank const &neighborRank : neighbors) // This does not include the current rank. + { + for (Face const &face : exchanged.at(neighborRank).faces) { - faceBuckets[{ curRank, neighborRank }].insert( *it ); - curFaces.erase( it ); + auto it = curFaces.find(face); + if (it != curFaces.cend()) + { + faceBuckets[{curRank, neighborRank}].insert(*it); + curFaces.erase(it); + } } } + faceBuckets[{curRank}] = curFaces; + + return {nodeBuckets, edgeBuckets, faceBuckets}; } - faceBuckets[{ curRank }] = curFaces; - return { nodeBuckets, edgeBuckets, faceBuckets }; -} + /** + * @brief Estimate an upper bound of the serialized size of the size @p buckets. + * @param buckets + * @return Size in bytes. + * @details @c MPI_Scan requires a fixed buffer size. + * An a-priori estimation of the maximum size of the buffer leads to crazy amounts of memory + * because of the combinatorial pattern of the rank intersections. + * Therefore, we use an initial MPI communication to get a better estimation. + * @node We don't care about the @c nodes because their global numbering is already done. + */ + std::size_t buildMaxBufferSize(Buckets const &buckets) // TODO give the size buckets instead? + { + // this is the custom MPI reduction function we use in the sum across ranks + auto const f = [](auto const &bucket) -> std::size_t + { + std::size_t size = std::size(bucket); + for (auto const &[ranks, _] : bucket) + { + size += std::size(ranks) + 1 + 1; // One `+1` for the size of the ranks set, the other one for the offset + } + return size; + }; + return MpiWrapper::sum(f(buckets.edges) + f(buckets.faces)); // Add the ScannedOffsets::{edge, face}Restart + } -/** - * @brief Estimate an upper bound of the serialized size of the size @p buckets. - * @param buckets - * @return Size in bytes. - * @details @c MPI_Scan requires a fixed buffer size. - * An a-priori estimation of the maximum size of the buffer leads to crazy amounts of memory - * because of the combinatorial pattern of the rank intersections. - * Therefore, we use an initial MPI communication to get a better estimation. - * @node We don't care about the @c nodes because their global numbering is already done. - */ -std::size_t buildMaxBufferSize( Buckets const & buckets ) // TODO give the size buckets instead? -{ - auto const f = []( auto const & bucket ) -> std::size_t + struct BucketSizes { - std::size_t size = std::size( bucket ); - for( auto const & [ranks, _]: bucket ) - { - size += std::size( ranks ) + 1 + 1; // One `+1` for the size of the ranks set, the other one for the offset - } - return size; + using mapping = std::map, localIndex>; + mapping edges; + mapping faces; }; - return MpiWrapper::sum( f( buckets.edges ) + f( buckets.faces ) ); // Add the ScannedOffsets::{edge, face}Restart -} + void to_json(json &j, + const BucketSizes &v) + { + j = json{{"edges", v.edges}, + {"faces", v.faces}}; + } -struct BucketSizes -{ - using mapping = std::map< std::set< MpiRank >, localIndex >; - mapping edges; - mapping faces; -}; + void from_json(const json &j, + BucketSizes &v) + { + v.edges = j.at("edges").get(); + v.faces = j.at("faces").get(); + } -void to_json( json & j, - const BucketSizes & v ) -{ - j = json{ { "edges", v.edges }, - { "faces", v.faces } }; -} + std::vector serialize(BucketSizes const &sizes) + { + return json::to_cbor(json(sizes)); + } -void from_json( const json & j, - BucketSizes & v ) -{ - v.edges = j.at( "edges" ).get< BucketSizes::mapping >(); - v.faces = j.at( "faces" ).get< BucketSizes::mapping >(); -} + std::vector serialize(BucketOffsets const &offsets) + { + return json::to_cbor(json(offsets)); + } -std::vector< std::uint8_t > serialize( BucketSizes const & sizes ) -{ - return json::to_cbor( json( sizes ) ); -} + /** + * @brief Compute the sizes of the intersection buckets. + * @param buckets The intersection buckets. + * @return The buckets of sizes. + */ + BucketSizes getBucketSize(Buckets const &buckets) + { + BucketSizes output; -std::vector< std::uint8_t > serialize( BucketOffsets const & offsets ) -{ - return json::to_cbor( json( offsets ) ); -} + for (auto const &[ranks, edges] : buckets.edges) + { + output.edges.emplace_hint(output.edges.end(), ranks, std::size(edges)); + } -/** - * @brief Compute the sizes of the intersection buckets. - * @param buckets The intersection buckets. - * @return The buckets of sizes. - */ -BucketSizes getBucketSize( Buckets const & buckets ) -{ - BucketSizes output; + for (auto const &[ranks, faces] : buckets.faces) + { + output.faces.emplace_hint(output.faces.end(), ranks, std::size(faces)); + } - for( auto const & [ranks, edges]: buckets.edges ) - { - output.edges.emplace_hint( output.edges.end(), ranks, std::size( edges ) ); + return output; } - for( auto const & [ranks, faces]: buckets.faces ) + /** + * @brief + * @tparam V Container of std::uint8_t + * @param data + * @return + */ + template + BucketOffsets deserialize(V const &data) { - output.faces.emplace_hint( output.faces.end(), ranks, std::size( faces ) ); + return json::from_cbor(data, false).template get(); } - return output; -} + /** + * @brief Custom MPI reduction function. Merges the sizes of the bucket from current rank into numbering offsets. + * @param[in] in Contains the reduced result from the previous ranks. Needs to be unpacked into a @c BucketOffsets instance. + * @param[inout] inout As an @e input, contains a pointer to the @c BucketSizes instance from the current rank. + * As an @e output, will contained the (packed) instance of @c BucketOffsets after the @c reduction. + * The @e output instance will be used by both current and next ranks. + * @param[in] len Number of data. + * @param[in] dataType Type of data. Mean to be @c MPI_BYTE for the current case. + */ + void f(void *in, + void *inout, + int *len, + MPI_Datatype *dataType) + { + GEOS_ASSERT_EQ(*dataType, MPI_BYTE); -/** - * @brief - * @tparam V Container of std::uint8_t - * @param data - * @return - */ -template< class V > -BucketOffsets deserialize( V const & data ) -{ - return json::from_cbor( data, false ).template get< BucketOffsets >(); -} - -/** - * @brief Custom MPI reduction function. Merges the sizes of the bucket from current rank into numbering offsets. - * @param[in] in Contains the reduced result from the previous ranks. Needs to be unpacked into a @c BucketOffsets instance. - * @param[inout] inout As an @e input, contains a pointer to the @c BucketSizes instance from the current rank. - * As an @e output, will contained the (packed) instance of @c BucketOffsets after the @c reduction. - * The @e output instance will be used by both current and next ranks. - * @param[in] len Number of data. - * @param[in] dataType Type of data. Mean to be @c MPI_BYTE for the current case. - */ -void f( void * in, - void * inout, - int * len, - MPI_Datatype * dataType ) -{ - GEOS_ASSERT_EQ( *dataType, MPI_BYTE ); + MpiRank const curRank{MpiWrapper::commRank()}; + + // offsets provided by the previous rank(s) + BucketOffsets const offsets = deserialize(Span((std::uint8_t *)in, *len)); + + // Sizes provided by the current rank, under the form of a pointer to the data. + // No need to serialize, since we're on the same rank. + std::uintptr_t addr; + std::memcpy(&addr, inout, sizeof(std::uintptr_t)); + BucketSizes const *sizes = reinterpret_cast(addr); - MpiRank const curRank{ MpiWrapper::commRank() }; + BucketOffsets updatedOffsets; + updatedOffsets.edges = updateBucketOffsets(sizes->edges, offsets.edges, curRank); + updatedOffsets.faces = updateBucketOffsets(sizes->faces, offsets.faces, curRank); - // offsets provided by the previous rank(s) - BucketOffsets const offsets = deserialize( Span< std::uint8_t >( (std::uint8_t *) in, *len ) ); + // Serialize the updated offsets, so they get sent to the next rank. + std::vector const serialized = serialize(updatedOffsets); + std::memcpy(inout, serialized.data(), serialized.size()); + } - // Sizes provided by the current rank, under the form of a pointer to the data. - // No need to serialize, since we're on the same rank. - std::uintptr_t addr; - std::memcpy( &addr, inout, sizeof( std::uintptr_t ) ); - BucketSizes const * sizes = reinterpret_cast(addr); + std::tuple doTheNewGlobalNumbering(vtkSmartPointer mesh, + std::set const &neighbors) + { + // Now we exchange the data with our neighbors. + MpiRank const curRank{MpiWrapper::commRank()}; - BucketOffsets updatedOffsets; - updatedOffsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes->edges, offsets.edges, curRank ); - updatedOffsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes->faces, offsets.faces, curRank ); + // ncs holds communicators to the neighboring ranks + std::vector ncs; + ncs.reserve(neighbors.size()); + for (MpiRank const &rank : neighbors) + { + ncs.emplace_back(rank.get()); + } - // Serialize the updated offsets, so they get sent to the next rank. - std::vector< std::uint8_t > const serialized = serialize( updatedOffsets ); - std::memcpy( inout, serialized.data(), serialized.size() ); -} + CommID const commId = CommunicationTools::getInstance().getCommID(); -std::tuple< Buckets, BucketOffsets > doTheNewGlobalNumbering( vtkSmartPointer< vtkDataSet > mesh, - std::set< MpiRank > const & neighbors ) -{ - // Now we exchange the data with our neighbors. - MpiRank const curRank{ MpiWrapper::commRank() }; + // On the current rank, we have a map from the neighbor MPI rank to a corresponding set of exchanged data (sets of nodes, edges, faces) + // buildExchangeData(mesh) just extracts the boundary (faces) of the current rank mesh + // exchange() actually does the communication, so that the output exchanged maps from nhbr rank to their boundary data + std::map exchanged = exchange(int(commId), ncs, buildExchangeData(mesh)); - std::vector< NeighborCommunicator > ncs; - ncs.reserve( neighbors.size() ); - for( MpiRank const & rank: neighbors ) - { - ncs.emplace_back( rank.get() ); - } + // Then I guess we just put the full mesh data for the current rank into the corresponding slot of exchanged, i think this is just convenient + exchanged[MpiRank{MpiWrapper::commRank()}] = buildFullData(mesh); - CommID const commId = CommunicationTools::getInstance().getCommID(); - std::map< MpiRank, Exchange > exchanged = exchange( int( commId ), ncs, buildExchangeData( mesh ) ); - exchanged[MpiRank{ MpiWrapper::commRank() }] = buildFullData( mesh ); + //std::cout << "exchanged on rank " << curRank << " -> " << json( exchanged[MpiRank{ MpiWrapper::commRank() }] ) << std::endl; -// std::cout << "exchanged on rank " << curRank << " -> " << json( exchanged[MpiRank{ MpiWrapper::commRank() }] ) << std::endl; + // Buckets - for each of (nodes, edges, faces), map from set of MPI ranks to the set of (nodes, edges, faces) shared by those ranks + Buckets const buckets = buildIntersectionBuckets(exchanged, curRank, neighbors); - Buckets const buckets = buildIntersectionBuckets( exchanged, curRank, neighbors ); + // Next step is communication. + // We will be passing around info about of the each ranks' buckets, and to do this we need to know the size of the data to be sent + // Technically this changes, as each rank has a different number of buckets + // For MPI scan, we need a fixed send/recv size, so we estimate with a large buffer size + // We dont want to do this too naively, as just taking the total possible cobinations of rank intersections leads to a massive size + // Instead, we do an MPI operation here to estimate an appropriate size + // The idea is to just sum the size of buckets across the ranks + std::size_t const maxBufferSize = 100 * buildMaxBufferSize(buckets); - std::size_t const maxBufferSize = 100 * buildMaxBufferSize( buckets ); + std::vector sendBuffer(maxBufferSize); + std::vector recvBuffer(maxBufferSize); - std::vector< std::uint8_t > sendBuffer( maxBufferSize ); - std::vector< std::uint8_t > recvBuffer( maxBufferSize ); + // The idea is that each rank needs to know where to start indexing the (nodes, edges, faces) it owns + // To do this, we need BucketOffsets, which are like buckets, but instead associate a global index to each set of MPI ranks, which marks the index of the first (node, edge, face) in that bucket + // To get this, we need to know the size of each bucket on the current rank, stored in BucketSizes + BucketSizes const sizes = getBucketSize(buckets); + BucketOffsets offsets; + if (curRank == 0_mpi) + { + // The `MPI_Scan` process will not call the reduction operator for rank 0. + // So we need to reduce for ourselves. + // updateBucketOffsets just takes the bucket sizes on the current rank, + // the previously computed offsets from other MPI ranks (none in this case), + // and then does the additions so that the global indices for the current buckets are known in offsets + offsets.edges = updateBucketOffsets(sizes.edges, {{{ + 0_mpi, + }, + 0_egi}}, + curRank); + offsets.faces = updateBucketOffsets(sizes.faces, {{{ + 0_mpi, + }, + 0_fgi}}, + curRank); + // Still we need to send this reduction to the following rank, by copying to it to the send buffer. + std::vector const bytes = serialize(offsets); + std::memcpy(sendBuffer.data(), bytes.data(), bytes.size()); + } + else + { + // For the other ranks, the reduction operator will be called during the `Mpi_Scan` process. + // So unlike for rank 0, we do not have to do it ourselves. + // In order to provide the `sizes` to the reduction operator, + // since `sizes` will only be used on the current rank, + // we'll provide the information as a pointer to the instance. + // The reduction operator will then compute the new offsets and send them to the following rank. + std::uintptr_t const addr = reinterpret_cast(&sizes); + std::memcpy(sendBuffer.data(), &addr, sizeof(std::uintptr_t)); + } - BucketSizes const sizes = getBucketSize( buckets ); - BucketOffsets offsets; - if( curRank == 0_mpi ) - { - // The `MPI_Scan` process will not call the reduction operator for rank 0. - // So we need to reduce ourselves for ourselves. - offsets.edges = updateBucketOffsets< EdgeGlbIdx >( sizes.edges, { { { 0_mpi, }, 0_egi } }, curRank ); - offsets.faces = updateBucketOffsets< FaceGlbIdx >( sizes.faces, { { { 0_mpi, }, 0_fgi } }, curRank ); - // Still we need to send this reduction to the following rank, by copying to it to the send buffer. - std::vector< std::uint8_t > const bytes = serialize( offsets ); - std::memcpy( sendBuffer.data(), bytes.data(), bytes.size() ); - } - else - { - // For the other ranks, the reduction operator will be called during the `Mpi_Scan` process. - // So unlike for rank 0, we do not have to do it ourselves. - // In order to provide the `sizes` to the reduction operator, - // since `sizes` will only be used on the current rank, - // we'll provide the information as a pointer to the instance. - // The reduction operator will then compute the new offsets and send them to the following rank. - std::uintptr_t const addr = reinterpret_cast(&sizes); - std::memcpy( sendBuffer.data(), &addr, sizeof( std::uintptr_t ) ); - } + // f is our mpi reduction function, all it essentially does is call the updateBucketOffsets as above, + // but it also handles the packing/unpacking of transferred data + // Note that, right now, packing/unpacking is done via json, this should be changed to GEOS packing routines + MPI_Op op; + MPI_Op_create(f, false, &op); - MPI_Op op; - MPI_Op_create( f, false, &op ); + MPI_Scan(sendBuffer.data(), recvBuffer.data(), maxBufferSize, MPI_BYTE, op, MPI_COMM_WORLD); - MPI_Scan( sendBuffer.data(), recvBuffer.data(), maxBufferSize, MPI_BYTE, op, MPI_COMM_WORLD ); + if (curRank != 0_mpi) + { + offsets = deserialize(recvBuffer); + } - if( curRank != 0_mpi ) - { - offsets = deserialize( recvBuffer ); + // With the offsets for each bucket, we are done with global numbering, + // as each (edge, face) is numbered by starting from the offset for that bucket and counting up + return {std::move(buckets), std::move(offsets)}; } - return { std::move( buckets ), std::move( offsets ) }; -} - } // end of namespace \ No newline at end of file From d9d06d2c7602d90dd1588806c363546229b55119 Mon Sep 17 00:00:00 2001 From: Ryan Aronson Date: Mon, 8 Jul 2024 18:20:18 +0000 Subject: [PATCH 102/106] initial attempt at adding cell type --- src/coreComponents/mesh/ElementType.hpp | 30 ++++++++++ .../mesh/generators/BuildPods.cpp | 55 ++++++++++++++----- .../mesh/generators/BuildPods.hpp | 2 +- .../mesh/generators/NewGhosting.cpp | 45 ++++++++++++--- .../mesh/generators/VTKUtilities.hpp | 3 + 5 files changed, 112 insertions(+), 23 deletions(-) diff --git a/src/coreComponents/mesh/ElementType.hpp b/src/coreComponents/mesh/ElementType.hpp index 8345080149f..a43f4328a37 100644 --- a/src/coreComponents/mesh/ElementType.hpp +++ b/src/coreComponents/mesh/ElementType.hpp @@ -89,6 +89,36 @@ inline int getElementDim( ElementType const elementType ) return 0; } +/** + * @brief Get number of faces for 3D elements + * @param elementType type of element + * @return number of faces for 3D elements (-1 if not a 3D element) + */ +inline int getNumFaces3D( ElementType const elementType ) +{ + switch( elementType ) + { + case ElementType::Vertex: + case ElementType::Line: + case ElementType::Triangle: + case ElementType::Quadrilateral: + case ElementType::Polygon: return -1; + case ElementType::Tetrahedron: return 4; + case ElementType::Pyramid: return 5; + case ElementType::Wedge: return 5; + case ElementType::Hexahedron: return 6; + case ElementType::Prism5: return 7; + case ElementType::Prism6: return 8; + case ElementType::Prism7: return 9; + case ElementType::Prism8: return 10; + case ElementType::Prism9: return 11; + case ElementType::Prism10: return 12; + case ElementType::Prism11: return 13; + case ElementType::Polyhedron: return -1; + } + return -1; +} + /// Strings for ElementType ENUM_STRINGS( ElementType, "Vertex", diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index acb25238665..69193411743 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -325,8 +325,11 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, // Building the `c2n` (cell to nodes), `c2e` (cell to edges) and `c2f` (cell to faces) mappings // loop thorugh meshgraph cells2face - for( auto const & [cgi, faceInfos]: graph.c2f ) + for( auto const & [cgi, typeAndFaces]: graph.c2f ) { + geos::ElementType const & cellType = std::get<0>(typeAndFaces); + std::vector const & faceInfos = std::get<1>(typeAndFaces); + // convert to cell local index CellLocIdx const & cli = g2l.cells.at( cgi ); @@ -354,21 +357,45 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, edges.assign( std::cbegin( tmpEdges ), std::cend( tmpEdges ) ); // c2n - // Note how we are hard-coded for hexs - FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type. - FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7) + // Here we need to implement all cell types + if (cellType == geos::ElementType::Hexahedron) + { + FaceInfo const & bottomFace = faceInfos.at( 4 ); // (0, 3, 2, 1) // TODO depends on element type. + FaceInfo const & topFace = faceInfos.at( 5 ); // (4, 5, 6, 7) + + std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( g2l.faces.at( bottomFace.index ) ); + std::vector< NodeLocIdx > const & topNodes = res.f2n.at( g2l.faces.at( topFace.index ) ); + + // reserFaceNodes resets the order using the flipped and start info + std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); + std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); + + // TODO carefully check the ordering... + // looks like this is geos order + std::array< NodeLocIdx, 8 > const tmp{ bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; + res.c2n[cli] = { tmp[0], tmp[1], tmp[3], tmp[2], tmp[4], tmp[5], tmp[7], tmp[6] }; + } + else if (cellType == geos::ElementType::Wedge) + { + FaceInfo const & bottomFace = faceInfos.at( 0 ); // (0, 1, 2) // TODO depends on element type. + FaceInfo const & topFace = faceInfos.at( 1 ); // (3, 5, 4) - std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( g2l.faces.at( bottomFace.index ) ); - std::vector< NodeLocIdx > const & topNodes = res.f2n.at( g2l.faces.at( topFace.index ) ); + std::vector< NodeLocIdx > const & bottomNodes = res.f2n.at( g2l.faces.at( bottomFace.index ) ); + std::vector< NodeLocIdx > const & topNodes = res.f2n.at( g2l.faces.at( topFace.index ) ); - // reserFaceNodes resets the order using the flipped and start info - std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); - std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); + // reserFaceNodes resets the order using the flipped and start info + std::vector< NodeLocIdx > const bn = resetFaceNodes( bottomNodes, bottomFace.isFlipped, bottomFace.start ); + std::vector< NodeLocIdx > const tn = resetFaceNodes( topNodes, topFace.isFlipped, topFace.start ); - // TODO carefully check the ordering... - // looks like this is geos order - std::array< NodeLocIdx, 8 > const tmp{ bn[0], bn[3], bn[2], bn[1], tn[0], tn[1], tn[2], tn[3] }; - res.c2n[cli] = { tmp[0], tmp[1], tmp[3], tmp[2], tmp[4], tmp[5], tmp[7], tmp[6] }; + // TODO carefully check the ordering... + // looks like this is geos order + std::array< NodeLocIdx, 6 > const tmp{ bn[0], bn[1], bn[2], tn[0], tn[2], tn[1] }; + res.c2n[cli] = { tmp[0], tmp[1], tmp[3], tmp[2], tmp[4], tmp[5]}; + } + else + { + GEOS_ERROR("c2n not implemented for this element type!"); + } } return res; @@ -531,7 +558,7 @@ void buildPods( MeshGraph const & owned, // Now we build the mappings used by GEOS // Downward - edges2nodes, faces2edges, faces2nodes, cells2faces, cells2edges, cells2nodes // The difference is that these use local indices, thus we built g2l - // there are assumptions that we are working on hexs here + // there are assumptions on what element types can be used here (to define c2n) DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); // invert to downward mappings to get e2f, f2c, n2e, n2f, n2c UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index fda8930c14e..95315e95c51 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -63,7 +63,7 @@ struct EdgeInfo struct MeshGraph { - std::map< CellGlbIdx, std::vector< FaceInfo > > c2f; + std::map< CellGlbIdx, std::tuple< geos::ElementType, std::vector< FaceInfo > > > c2f; // adding in element type std::map< FaceGlbIdx, std::vector< EdgeInfo > > f2e; std::map< EdgeGlbIdx, std::tuple< NodeGlbIdx, NodeGlbIdx > > e2n; // TODO use Edge here? std::map< NodeGlbIdx, std::array< double, 3 > > n2pos; diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 953801b6541..1e6d36a2855 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -36,6 +36,8 @@ #include +#include "VTKUtilities.hpp" // this may be a bad include..., used for converter of vtk to element type + using json = nlohmann::json; #include @@ -44,6 +46,27 @@ using json = nlohmann::json; namespace geos::ghosting { + /** + * THIS IS DUPLICATED, SHOULD USE VTK UTILITIES, BUT ENDED UP WITH LINKER ERROR (MULTIPLY DEFINED) + * but also dont know what we want to do for non-vtk mesh + */ +ElementType convertVtkToGeosxElementTypeGhosting( vtkCell *cell ) +{ + switch( cell->GetCellType() ) + { + case VTK_WEDGE: return ElementType::Wedge; + case VTK_VOXEL: return ElementType::Hexahedron; + case VTK_HEXAHEDRON: return ElementType::Hexahedron; + default: + { + GEOS_ERROR( cell->GetCellType() << " is not a recognized cell type in ghosting.\n" << + generalMeshErrorAdvice ); + return {}; + } + } +} + + struct MaxGlbIdcs { NodeGlbIdx nodes; @@ -176,7 +199,7 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > MpiRank curRank ) { // MeshGraph (defined in BuildPods.hpp) is just a struct of maps which map: - // cell global index to vector of entries of type (face global index, extra stuff defining orientation of face) for each of the adjacent faces + // cell global index to tuple of element type, and vector of entries of type (face global index, extra stuff defining orientation of face) for each of the adjacent faces // face global index to vector of entries of type (edge global index plus orientation) for each of the adjacent edges // edge global index to tuple of node global indices // node global index to 3d array of position @@ -275,6 +298,7 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > vtkIdTypeArray const * globalCellIds = vtkIdTypeArray::FastDownCast( mesh->GetCellData()->GetGlobalIds() ); // TODO do the mapping beforehand for( vtkIdType c = 0; c < mesh->GetNumberOfCells(); ++c ) { + std::vector< FaceInfo > faces; vtkCell * cell = mesh->GetCell( c ); CellGlbIdx const gci{ globalCellIds->GetValue( c ) }; // TODO copy paste? @@ -301,7 +325,7 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > // add the face data to the vector for this cell in owned graph. // note there will never be any 'present' cells, as the earlier exchange only communicates the 2D boundary btwn ranks // note we are using the n2f built above to go from the vector of nodes (properly ordered) to the face global ID - owned.c2f[gci].emplace_back( FaceInfo{ n2f.at( reorderedFaceNodes ), isFlipped, start } ); + owned.c2f[gci] = {convertVtkToGeosxElementTypeGhosting(cell), faces}; // from vtkUtilities, had to duplicate } } @@ -694,8 +718,11 @@ Adjacency buildAdjacency( MeshGraph const & owned, ind.back() = ownedGlbIdcs.back(); val.back() = intConv< TriScalarInt >( numEdges ); } - for( auto const & [cgi, faces]: owned.c2f ) + for( auto const & [cgi, typeAndFaces]: owned.c2f ) { + geos::ElementType const & cellType = std::get<0>(typeAndFaces); + std::vector const & faces = std::get<1>(typeAndFaces); + // The same comment as for faces and edges applies for cells and faces (see above). // The main differences being that // - the `FaceInfo` as a little more information, @@ -726,7 +753,7 @@ Adjacency buildAdjacency( MeshGraph const & owned, } // add diagonal data ind.back() = ownedGlbIdcs.back(); - val.back() = intConv< TriScalarInt >( numFaces ); // TODO This should be Hex and the not the number of faces... + val.back() = intConv< TriScalarInt >( static_cast(cellType) ); // TODO This should be Hex and the not the number of faces... } GEOS_ASSERT_EQ( numOwned, std::size( ownedGlbIdcs ) ); @@ -1252,7 +1279,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // Get the diagonal value std::size_t const numGeomQuantitiesIdx = intConv< std::size_t >( std::distance( std::cbegin( extractedIndices ), cit ) ); TriScalarInt const & numGeomQuantities = extractedValues[numGeomQuantitiesIdx]; - GEOS_ASSERT_EQ( extracted, intConv< std::size_t >( numGeomQuantities + 1 ) ); // extracted is the output from Tilinos telling you how many entries were pulled out when we copy row + //GEOS_ASSERT_EQ( extracted, intConv< std::size_t >( numGeomQuantities + 1 ) ); // extracted is the output from Tilinos telling you how many entries were pulled out when we copy row, no longer true for cells switch( geometricalType ) { @@ -1322,7 +1349,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } case Geom::CELL: { - TriScalarInt const & numFaces = numGeomQuantities; // Alias // TODO This should receive the cell type instead. + geos::ElementType const cellType = static_cast(numGeomQuantities); + int const & numFaces = getNumFaces3D(cellType); std::map< integer, FaceInfo > order; // loop over the entries extracted from the row of the matrix @@ -1345,7 +1373,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // use cell global index and `order` to populate ghosts CellGlbIdx const cgi = convert.toCellGlbIdx( index ); - std::vector< FaceInfo > & tmp = ghosts.c2f[cgi]; + std::get<0>(ghosts.c2f[cgi]) = cellType; + std::vector< FaceInfo > & tmp = std::get<1>(ghosts.c2f[cgi]); tmp.resize( numFaces ); for( auto const & [ord, faceInfo]: order ) { @@ -1410,7 +1439,7 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, // Now we build to data to describe the mesh data on the current rank. // owned and present are struct of maps which map: - // cell global index to vector of entries of type (face global index, extra stuff defining orientation of face) for each of the adjacent faces + // cell global index to tuple of cell type and vector of entries of type (face global index, extra stuff defining orientation of face) for each of the adjacent faces // face global index to vector of entries of type (edge global index plus orientation) for each of the adjacent edges // edge global index to tuple of node global indices // node global index to 3d array of position diff --git a/src/coreComponents/mesh/generators/VTKUtilities.hpp b/src/coreComponents/mesh/generators/VTKUtilities.hpp index 6f8e97a2180..82abf420d78 100644 --- a/src/coreComponents/mesh/generators/VTKUtilities.hpp +++ b/src/coreComponents/mesh/generators/VTKUtilities.hpp @@ -27,6 +27,9 @@ #include #include +#include // I just needed this to get vtkCell definition + + #include #include From 020955f8a29ce38d7d9b874bf70938133b67406e Mon Sep 17 00:00:00 2001 From: Ryan Aronson Date: Mon, 8 Jul 2024 22:12:12 +0000 Subject: [PATCH 103/106] fixed bug for hex meshes with elem type --- src/coreComponents/mesh/generators/BuildPods.cpp | 8 +++++++- src/coreComponents/mesh/generators/NewGhosting.cpp | 11 ++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 69193411743..706c6f19eec 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -324,7 +324,7 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, } // Building the `c2n` (cell to nodes), `c2e` (cell to edges) and `c2f` (cell to faces) mappings - // loop thorugh meshgraph cells2face + // loop through meshgraph cells2face for( auto const & [cgi, typeAndFaces]: graph.c2f ) { geos::ElementType const & cellType = std::get<0>(typeAndFaces); @@ -545,6 +545,8 @@ void buildPods( MeshGraph const & owned, // start by merging the 3 mesh graphs for the different types of data on the graph into 1 struct MeshGraph const graph = mergeMeshGraph( owned, present, ghosts ); + GEOS_LOG_RANK("Merged the mesh graphs"); + // create GlobalToLocal, which maps global IDs to rank local IDs for (nodes, edges, faces, cells) // mapKeys is just a utility which extracts keys // buildGlobalToLocalMap just takes the global ids and maps them to 1, 2, 3, ... @@ -563,6 +565,8 @@ void buildPods( MeshGraph const & owned, // invert to downward mappings to get e2f, f2c, n2e, n2f, n2c UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); + GEOS_LOG_RANK("Built upward and downward mappings"); + // Now we can make our node/edge/face/cell manager // NodeMgrImpl inherits from nodeManager (see pods.hpp) (interface with getters and setters) // We basically have all the data, this function just casts it into the proper GEOS types @@ -598,6 +602,8 @@ void buildPods( MeshGraph const & owned, invertGhostRecv( recv.cells, g2l.cells ) ) ) ); meshMappings.setNeighbors( getNeighbors( recv, send ) ); + + GEOS_LOG_RANK("built mesh mappings"); } } diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 1e6d36a2855..533875406ea 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -321,12 +321,12 @@ std::tuple< MeshGraph, MeshGraph > buildMeshGraph( vtkSmartPointer< vtkDataSet > bool isFlipped; std::uint8_t start; std::vector< NodeGlbIdx > const reorderedFaceNodes = reorderFaceNodes( faceNodes, isFlipped, start ); - // add the face data to the vector for this cell in owned graph. // note there will never be any 'present' cells, as the earlier exchange only communicates the 2D boundary btwn ranks // note we are using the n2f built above to go from the vector of nodes (properly ordered) to the face global ID - owned.c2f[gci] = {convertVtkToGeosxElementTypeGhosting(cell), faces}; // from vtkUtilities, had to duplicate + faces.emplace_back(FaceInfo{n2f.at(reorderedFaceNodes), isFlipped, start}); } + owned.c2f[gci] = {convertVtkToGeosxElementTypeGhosting(cell), faces}; // from vtkUtilities, had to duplicate } GEOS_ASSERT( std::empty( present.c2f ) ); @@ -937,7 +937,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & GEOS_LOG_RANK( "ghostExchange->NumGlobalCols() = " << ghostExchange.getGlobalNumCols() ); GEOS_LOG_RANK( "ghostExchange->NumGlobalRows() = " << ghostExchange.getGlobalNumRows() ); } - Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ghostInfo.mat", ghostExchange ); + // Tpetra::MatrixMarket::Writer< TriCrsMatrix >::writeSparseFile( "/tmp/matrices/ghostInfo.mat", ghostExchange ); // Now, for each of the entities owned by this current rank, we collect all the other ranks to which this entity is sent @@ -1011,6 +1011,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } } // end loop over owned entities + GEOS_LOG_RANK("Created Ghost Send"); + // Now we can look at the dual problem, namely for this current rank, what data is needed from other ranks // Now we also have to pay attention to what non-owned data was already present on the rank @@ -1099,6 +1101,8 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & } } + GEOS_LOG_RANK("Created Ghost Receive"); + // debug output // if( curRank == 1_mpi or curRank == 2_mpi ) // { @@ -1410,6 +1414,7 @@ std::tuple< MeshGraph, GhostRecv, GhostSend > performGhosting( MeshGraph const & // } // Now we are done!!! + GEOS_LOG_RANK("Done with ghosting"); return { ghosts, recv, send }; } From a62ce3ae5f2a882a81c46f67b104860e2bec7dd5 Mon Sep 17 00:00:00 2001 From: Ryan Aronson Date: Thu, 11 Jul 2024 21:05:10 +0000 Subject: [PATCH 104/106] attempt at mutliple cell types, need to debug --- src/coreComponents/mesh/ElementType.hpp | 60 ++++++ .../mesh/generators/BuildPods.cpp | 182 ++++++++++++++++-- .../mesh/generators/BuildPods.hpp | 4 +- .../mesh/generators/NewGhosting.cpp | 5 +- src/coreComponents/mesh/generators/Pods.cpp | 19 +- src/coreComponents/mesh/generators/Pods.hpp | 8 +- 6 files changed, 255 insertions(+), 23 deletions(-) diff --git a/src/coreComponents/mesh/ElementType.hpp b/src/coreComponents/mesh/ElementType.hpp index a43f4328a37..b046934ca88 100644 --- a/src/coreComponents/mesh/ElementType.hpp +++ b/src/coreComponents/mesh/ElementType.hpp @@ -119,6 +119,66 @@ inline int getNumFaces3D( ElementType const elementType ) return -1; } +/** + * @brief Get number of edges for 3D elements + * @param elementType type of element + * @return number of edges for 3D elements (-1 if not a 3D element) + */ +inline int getNumEdges3D( ElementType const elementType ) +{ + switch( elementType ) + { + case ElementType::Vertex: + case ElementType::Line: + case ElementType::Triangle: + case ElementType::Quadrilateral: + case ElementType::Polygon: return -1; + case ElementType::Tetrahedron: return 6; + case ElementType::Pyramid: return 8; + case ElementType::Wedge: return 9; + case ElementType::Hexahedron: return 12; + case ElementType::Prism5: return -1; + case ElementType::Prism6: return -1; + case ElementType::Prism7: return -1; + case ElementType::Prism8: return -1; + case ElementType::Prism9: return -1; + case ElementType::Prism10: return -1; + case ElementType::Prism11: return -1; + case ElementType::Polyhedron: return -1; + } + return -1; +} + +/** + * @brief Get number of nodes for 3D elements + * @param elementType type of element + * @return number of modes for 3D elements (-1 if not a 3D element) + */ +inline int getNumNodes3D( ElementType const elementType ) +{ + switch( elementType ) + { + case ElementType::Vertex: + case ElementType::Line: + case ElementType::Triangle: + case ElementType::Quadrilateral: + case ElementType::Polygon: return -1; + case ElementType::Tetrahedron: return 4; + case ElementType::Pyramid: return 5; + case ElementType::Wedge: return 6; + case ElementType::Hexahedron: return 8; + case ElementType::Prism5: return -1; + case ElementType::Prism6: return -1; + case ElementType::Prism7: return -1; + case ElementType::Prism8: return -1; + case ElementType::Prism9: return -1; + case ElementType::Prism10: return -1; + case ElementType::Prism11: return -1; + case ElementType::Polyhedron: return -1; + } + return -1; +} + /// Strings for ElementType ENUM_STRINGS( ElementType, "Vertex", diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 706c6f19eec..b8209f56994 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -34,6 +34,32 @@ struct GlobalToLocal std::map< CellGlbIdx, CellLocIdx > cells; }; +struct LocalToGlobal +{ + std::map< NodeLocIdx, NodeGlbIdx > nodes; + std::map< EdgeLocIdx, EdgeGlbIdx > edges; + std::map< FaceLocIdx, FaceGlbIdx > faces; + std::map< CellLocIdx, CellGlbIdx > cells; +}; + + /** + * THIS IS DUPLICATED, SHOULD USE VTK UTILITIES, BUT ENDED UP WITH LINKER ERROR (MULTIPLY DEFINED) + * but also dont know what we want to do for non-vtk mesh + */ +std::string convertGeosxElementStringVtkStringGhosting( ElementType const & cellType) +{ + switch( cellType) + { + case ElementType::Hexahedron: return string("hexahedra"); + case ElementType::Wedge: return string("wedges"); + default: + { + GEOS_ERROR( cellType << " is not a recognized cell type in ghosting.\n" ); + return {}; + } + } +} + /** * @brief Computes a global to local map from the global indices provided in @c gis. * @tparam GI The global index type. @@ -55,6 +81,27 @@ std::map< GI, toLocIdx_t< GI > > buildGlobalToLocalMap( std::set< GI > const & g return g2l; } +// TODO I think thomas was doing this (implicitly), I needed it for splitting into cell types +/** + * @brief Computes a local to global map by inverting global to local + * @tparam + * @param + * @return The map instance. + */ +template< class GI > +std::map< toLocIdx_t< GI >, GI > buildLocalToGlobalMap( std::map< GI, toLocIdx_t< GI > > const & g2l ) +{ + using LI = toLocIdx_t< GI >; + std::map< LI, GI > l2g; + + for (auto const & [gi, li] : g2l) + { + l2g.insert({li, gi}); + } + + return l2g; +} + // Consolidate the mesh graphs on the current rank into one object MeshGraph mergeMeshGraph( MeshGraph const & owned, MeshGraph const & present, @@ -81,6 +128,7 @@ struct DownwardMappings std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n; std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f; std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; + std::map< CellLocIdx, geos::ElementType > c2t; }; struct UpwardMappings @@ -90,6 +138,7 @@ struct UpwardMappings std::map< NodeLocIdx, std::vector< EdgeLocIdx > > n2e; std::map< NodeLocIdx, std::vector< FaceLocIdx > > n2f; std::map< NodeLocIdx, std::vector< CellLocIdx > > n2c; + std::map< geos::ElementType, std::vector< CellLocIdx > > t2c; }; // converts generic map to arrayOfArrays @@ -262,7 +311,8 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, toFlavorlessMapping( recv ) ); } -CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, +CellBlkImpl makeFlavorlessCellBlkImpl( ElementType cellType, + std::size_t const & numCells, std::map< CellLocIdx, std::vector< NodeLocIdx > > const & c2n, std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, @@ -275,10 +325,16 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); - return CellBlkImpl( intConv< localIndex >( numCells ), - convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, 8 ), - convertToA2d( c2e, 12 ), - convertToA2d( c2f, 6 ), + // get num nodes edges faces per cell from type + int numNodesPerCell = getNumNodes3D(cellType); + int numEdgesPerCell = getNumEdges3D(cellType); + int numFacesPerCell = getNumFaces3D(cellType); + + return CellBlkImpl( cellType, + intConv< localIndex >( numCells ), + convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, numNodesPerCell ), + convertToA2d( c2e, numEdgesPerCell ), + convertToA2d( c2f, numFacesPerCell ), convertGlobalToLocalMap( cg2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); @@ -333,6 +389,8 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, // convert to cell local index CellLocIdx const & cli = g2l.cells.at( cgi ); + res.c2t[cli] = cellType; // may have been easier to do this with global indices + std::vector< FaceLocIdx > & faces = res.c2f[cli]; std::vector< EdgeLocIdx > & edges = res.c2e[cli]; @@ -453,6 +511,12 @@ UpwardMappings buildUpwardMappings( DownwardMappings const & downwardMappings ) } } + // Filling `t2c` + for( auto const & [cli, type]: downwardMappings.c2t ) + { + res.t2c[type].emplace_back( cli ); // may have been easier to do this with global indices + } + return res; } @@ -535,11 +599,87 @@ std::map< MpiRank, std::vector< LI > > invertGhostRecv( std::map< GI, MpiRank > return output; } +CellBlkImpl makeFlavorlessCellBlkImpls( std::size_t const & numCells, + std::map< CellLocIdx, std::vector< NodeLocIdx > > & c2n, + std::map< CellLocIdx, std::vector< EdgeLocIdx > > & c2e, + std::map< CellLocIdx, std::vector< FaceLocIdx > > & c2f, + std::map< CellGlbIdx, CellLocIdx > & cg2l, + std::map< CellGlbIdx, std::set< MpiRank > > & send, + std::map< CellGlbIdx, MpiRank > & recv, + std::map< geos::ElementType, std::vector< CellLocIdx > > & t2c, + std::map< CellLocIdx, CellGlbIdx > & cl2g ) +{ + GEOS_ASSERT_EQ( numCells, std::size( c2n ) ); + GEOS_ASSERT_EQ( numCells, std::size( c2e ) ); + GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); + GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); + + std::map< string, CellBlkImpl > ret; + + for (auto const & [cellType, clis]: t2c) + { + // if (cellType == ElementType::Wedge) {continue;} + // create structs that will be populated for this cell + string blkName = convertGeosxElementStringVtkStringGhosting(cellType); + std::size_t numCells_block = clis.size(); + std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n_block; + std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e_block; + std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f_block; + std::map< CellGlbIdx, CellLocIdx > cg2l_block; + std::map< CellGlbIdx, std::set< MpiRank > > send_block; + std::map< CellGlbIdx, MpiRank > recv_block; + + GEOS_LOG_RANK("building block of type " << cellType << " / " << blkName); + + // populate data structs for current cell type + for (auto const & cli: clis) + { + c2n_block[cli] = c2n[cli]; + c2e_block[cli] = c2e[cli]; + c2f_block[cli] = c2f[cli]; + //need the global index of this local index + cg2l_block[cl2g[cli]] = cli; + send_block[cl2g[cli]] = send[cl2g[cli]]; + recv_block[cl2g[cli]] = recv[cl2g[cli]]; + } + + // add name + CellBlkImpl (call constructor with new data) to ret map + // ret[blkName] = makeFlavorlessCellBlkImpl( cellType, + // numCells_block, + // c2n_block, + // c2e_block, + // c2f_block, + // cg2l_block, + // invertGhostSend(send_block, cg2l_block), + // invertGhostRecv(recv_block, cg2l_block) ); + + // ret[blkName] = makeFlavorlessCellBlkImpl( cellType, + // numCells, + // c2n, + // c2e, + // c2f, + // cg2l, + // invertGhostSend(send, cg2l), + // invertGhostRecv(recv, cg2l) ); + + // GEOS_LOG_RANK("Finished block of type " << cellType << " / " << blkName); + } + + return makeFlavorlessCellBlkImpl( ElementType::Hexahedron, + numCells, + c2n, + c2e, + c2f, + cg2l, + invertGhostSend(send, cg2l), + invertGhostRecv(recv, cg2l) ); +} + void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, - GhostRecv const & recv, - GhostSend const & send, + GhostRecv & recv, + GhostSend & send, MeshMappingImpl & meshMappings ) { // start by merging the 3 mesh graphs for the different types of data on the graph into 1 struct @@ -550,20 +690,28 @@ void buildPods( MeshGraph const & owned, // create GlobalToLocal, which maps global IDs to rank local IDs for (nodes, edges, faces, cells) // mapKeys is just a utility which extracts keys // buildGlobalToLocalMap just takes the global ids and maps them to 1, 2, 3, ... - GlobalToLocal const g2l{ + GlobalToLocal g2l{ buildGlobalToLocalMap( mapKeys< std::set >( graph.n2pos ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.e2n ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.f2e ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.c2f ) ) }; + // invert GLobal to local, as I needed one thing for multiple cell types + LocalToGlobal l2g{ + buildLocalToGlobalMap( g2l.nodes ), + buildLocalToGlobalMap( g2l.edges ), + buildLocalToGlobalMap( g2l.faces ), + buildLocalToGlobalMap( g2l.cells ) + }; + // Now we build the mappings used by GEOS // Downward - edges2nodes, faces2edges, faces2nodes, cells2faces, cells2edges, cells2nodes // The difference is that these use local indices, thus we built g2l // there are assumptions on what element types can be used here (to define c2n) - DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); + DownwardMappings downwardMappings = buildDownwardMappings( g2l, graph ); // invert to downward mappings to get e2f, f2c, n2e, n2f, n2c - UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); + UpwardMappings upwardMappings = buildUpwardMappings( downwardMappings ); GEOS_LOG_RANK("Built upward and downward mappings"); @@ -593,13 +741,21 @@ void buildPods( MeshGraph const & owned, g2l.nodes, invertGhostSend( send.nodes, g2l.nodes ), invertGhostRecv( recv.nodes, g2l.nodes ) ) ); - meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpl( std::size( g2l.cells ), + + + // here we need to make multiple cell blocks, instead of just one + // CellMgrImpl should accept multiple cell blocks, so that it can return them + // then the getCellBlocks should just return the member variable for CellMgr + // makeFlavorlessCellBlkImpl returns a CellBlkImpl, so a new version should return whatever CellMgrImpl wants + meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpls( std::size( g2l.cells ), downwardMappings.c2n, downwardMappings.c2e, downwardMappings.c2f, g2l.cells, - invertGhostSend( send.cells, g2l.cells ), - invertGhostRecv( recv.cells, g2l.cells ) ) ) ); + send.cells, + recv.cells, + upwardMappings.t2c, + l2g.cells ) ) ); meshMappings.setNeighbors( getNeighbors( recv, send ) ); diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index 95315e95c51..6110fd7e1af 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -88,8 +88,8 @@ struct GhostRecv void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, - GhostRecv const & recv, - GhostSend const & send, + GhostRecv & recv, + GhostSend & send, MeshMappingImpl & meshMappings ); } diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 533875406ea..68ade942c39 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -59,8 +59,7 @@ ElementType convertVtkToGeosxElementTypeGhosting( vtkCell *cell ) case VTK_HEXAHEDRON: return ElementType::Hexahedron; default: { - GEOS_ERROR( cell->GetCellType() << " is not a recognized cell type in ghosting.\n" << - generalMeshErrorAdvice ); + GEOS_ERROR( cell->GetCellType() << " is not a recognized cell type in ghosting.\n" ); return {}; } } @@ -1461,7 +1460,7 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, // By doing some clever matrix products we can figure out which ranks need what automagically // `ghosts` is also a MeshGraph like owned and present, but described the geometric entities which are ghosted onto the rank from another owner // recv and send are of type GhostRecv and GhostSend (in BuildPods.hpp) and describe what (nodes, edges, faces, cells) each rank needs to receive from and send to others - auto const [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); + auto [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); // Finally, we use everything we have to populate the mappings that interface with the rest of GEOS // Note that we have already done the ghosting, so these mappings are set once and for all diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index c9aa3f8ff34..ada17439485 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -91,7 +91,8 @@ ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const return { std::move( toBlockIndex ), m_f2c }; } -CellBlkImpl::CellBlkImpl( localIndex numCells, +CellBlkImpl::CellBlkImpl( ElementType cellType, + localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -102,32 +103,44 @@ CellBlkImpl::CellBlkImpl( localIndex numCells, m_numCells( numCells ), m_c2n( c2n ), m_c2e( c2e ), - m_c2f( c2f ) + m_c2f( c2f ), + m_cellType( cellType ) { } ElementType CellBlkImpl::getElementType() const { + // return m_cellType; return ElementType::Hexahedron; } localIndex CellBlkImpl::numNodesPerElement() const { + // return m_c2n.size(1); return 8; } localIndex CellBlkImpl::numEdgesPerElement() const { + // return m_c2e.size(1); return 12; } localIndex CellBlkImpl::numFacesPerElement() const { + // return m_c2f.size(1); return 6; } std::map< string, generators::CellBlk const * > CellMgrImpl::getCellBlks() const { - return { { string( "hexahedra" ), &m_cellBlk } }; // TODO hard coded values. + // std::map< string, generators::CellBlk const * > ret; + + // for (auto const & [blkName, cellBlk] : m_cellBlks) + // { + // ret[blkName] = & cellBlk; + // } + + return { { string( "hexahedra" ), &m_cellBlk } }; } } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 6a22754d5b1..ab3b91ee5f0 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -209,7 +209,8 @@ class CellBlkImpl : public generators::CellBlk public: CellBlkImpl() = default; - CellBlkImpl( localIndex numCells, + CellBlkImpl( ElementType cellType, + localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -266,6 +267,7 @@ class CellBlkImpl : public generators::CellBlk array2d< localIndex, cells::NODE_MAP_PERMUTATION > m_c2n; array2d< localIndex > m_c2e; array2d< localIndex > m_c2f; + ElementType m_cellType; }; class CellMgrImpl : public generators::CellMgr @@ -273,6 +275,7 @@ class CellMgrImpl : public generators::CellMgr public: CellMgrImpl() = default; + // CellMgrImpl( std::map< string, CellBlkImpl > && cellBlks ) CellMgrImpl( CellBlkImpl && cellBlks ) : m_cellBlk( cellBlks ) @@ -281,7 +284,8 @@ class CellMgrImpl : public generators::CellMgr [[nodiscard]] std::map< string, generators::CellBlk const * > getCellBlks() const override; private: - CellBlkImpl m_cellBlk; + // std::map< string, CellBlkImpl > m_cellBlks; + CellBlkImpl m_cellBlk; }; class MeshMappingImpl : public generators::MeshMappings From ba0f8892c6c1a8666814e1c9f5020d83f983701f Mon Sep 17 00:00:00 2001 From: Ryan Aronson Date: Thu, 11 Jul 2024 21:53:53 +0000 Subject: [PATCH 105/106] Revert "attempt at mutliple cell types, need to debug" This reverts commit a62ce3ae5f2a882a81c46f67b104860e2bec7dd5. --- src/coreComponents/mesh/ElementType.hpp | 60 ------ .../mesh/generators/BuildPods.cpp | 182 ++---------------- .../mesh/generators/BuildPods.hpp | 4 +- .../mesh/generators/NewGhosting.cpp | 5 +- src/coreComponents/mesh/generators/Pods.cpp | 19 +- src/coreComponents/mesh/generators/Pods.hpp | 8 +- 6 files changed, 23 insertions(+), 255 deletions(-) diff --git a/src/coreComponents/mesh/ElementType.hpp b/src/coreComponents/mesh/ElementType.hpp index b046934ca88..a43f4328a37 100644 --- a/src/coreComponents/mesh/ElementType.hpp +++ b/src/coreComponents/mesh/ElementType.hpp @@ -119,66 +119,6 @@ inline int getNumFaces3D( ElementType const elementType ) return -1; } -/** - * @brief Get number of edges for 3D elements - * @param elementType type of element - * @return number of edges for 3D elements (-1 if not a 3D element) - */ -inline int getNumEdges3D( ElementType const elementType ) -{ - switch( elementType ) - { - case ElementType::Vertex: - case ElementType::Line: - case ElementType::Triangle: - case ElementType::Quadrilateral: - case ElementType::Polygon: return -1; - case ElementType::Tetrahedron: return 6; - case ElementType::Pyramid: return 8; - case ElementType::Wedge: return 9; - case ElementType::Hexahedron: return 12; - case ElementType::Prism5: return -1; - case ElementType::Prism6: return -1; - case ElementType::Prism7: return -1; - case ElementType::Prism8: return -1; - case ElementType::Prism9: return -1; - case ElementType::Prism10: return -1; - case ElementType::Prism11: return -1; - case ElementType::Polyhedron: return -1; - } - return -1; -} - -/** - * @brief Get number of nodes for 3D elements - * @param elementType type of element - * @return number of modes for 3D elements (-1 if not a 3D element) - */ -inline int getNumNodes3D( ElementType const elementType ) -{ - switch( elementType ) - { - case ElementType::Vertex: - case ElementType::Line: - case ElementType::Triangle: - case ElementType::Quadrilateral: - case ElementType::Polygon: return -1; - case ElementType::Tetrahedron: return 4; - case ElementType::Pyramid: return 5; - case ElementType::Wedge: return 6; - case ElementType::Hexahedron: return 8; - case ElementType::Prism5: return -1; - case ElementType::Prism6: return -1; - case ElementType::Prism7: return -1; - case ElementType::Prism8: return -1; - case ElementType::Prism9: return -1; - case ElementType::Prism10: return -1; - case ElementType::Prism11: return -1; - case ElementType::Polyhedron: return -1; - } - return -1; -} - /// Strings for ElementType ENUM_STRINGS( ElementType, "Vertex", diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index b8209f56994..706c6f19eec 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -34,32 +34,6 @@ struct GlobalToLocal std::map< CellGlbIdx, CellLocIdx > cells; }; -struct LocalToGlobal -{ - std::map< NodeLocIdx, NodeGlbIdx > nodes; - std::map< EdgeLocIdx, EdgeGlbIdx > edges; - std::map< FaceLocIdx, FaceGlbIdx > faces; - std::map< CellLocIdx, CellGlbIdx > cells; -}; - - /** - * THIS IS DUPLICATED, SHOULD USE VTK UTILITIES, BUT ENDED UP WITH LINKER ERROR (MULTIPLY DEFINED) - * but also dont know what we want to do for non-vtk mesh - */ -std::string convertGeosxElementStringVtkStringGhosting( ElementType const & cellType) -{ - switch( cellType) - { - case ElementType::Hexahedron: return string("hexahedra"); - case ElementType::Wedge: return string("wedges"); - default: - { - GEOS_ERROR( cellType << " is not a recognized cell type in ghosting.\n" ); - return {}; - } - } -} - /** * @brief Computes a global to local map from the global indices provided in @c gis. * @tparam GI The global index type. @@ -81,27 +55,6 @@ std::map< GI, toLocIdx_t< GI > > buildGlobalToLocalMap( std::set< GI > const & g return g2l; } -// TODO I think thomas was doing this (implicitly), I needed it for splitting into cell types -/** - * @brief Computes a local to global map by inverting global to local - * @tparam - * @param - * @return The map instance. - */ -template< class GI > -std::map< toLocIdx_t< GI >, GI > buildLocalToGlobalMap( std::map< GI, toLocIdx_t< GI > > const & g2l ) -{ - using LI = toLocIdx_t< GI >; - std::map< LI, GI > l2g; - - for (auto const & [gi, li] : g2l) - { - l2g.insert({li, gi}); - } - - return l2g; -} - // Consolidate the mesh graphs on the current rank into one object MeshGraph mergeMeshGraph( MeshGraph const & owned, MeshGraph const & present, @@ -128,7 +81,6 @@ struct DownwardMappings std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n; std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f; std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; - std::map< CellLocIdx, geos::ElementType > c2t; }; struct UpwardMappings @@ -138,7 +90,6 @@ struct UpwardMappings std::map< NodeLocIdx, std::vector< EdgeLocIdx > > n2e; std::map< NodeLocIdx, std::vector< FaceLocIdx > > n2f; std::map< NodeLocIdx, std::vector< CellLocIdx > > n2c; - std::map< geos::ElementType, std::vector< CellLocIdx > > t2c; }; // converts generic map to arrayOfArrays @@ -311,8 +262,7 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, toFlavorlessMapping( recv ) ); } -CellBlkImpl makeFlavorlessCellBlkImpl( ElementType cellType, - std::size_t const & numCells, +CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, std::map< CellLocIdx, std::vector< NodeLocIdx > > const & c2n, std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, @@ -325,16 +275,10 @@ CellBlkImpl makeFlavorlessCellBlkImpl( ElementType cellType, GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); - // get num nodes edges faces per cell from type - int numNodesPerCell = getNumNodes3D(cellType); - int numEdgesPerCell = getNumEdges3D(cellType); - int numFacesPerCell = getNumFaces3D(cellType); - - return CellBlkImpl( cellType, - intConv< localIndex >( numCells ), - convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, numNodesPerCell ), - convertToA2d( c2e, numEdgesPerCell ), - convertToA2d( c2f, numFacesPerCell ), + return CellBlkImpl( intConv< localIndex >( numCells ), + convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, 8 ), + convertToA2d( c2e, 12 ), + convertToA2d( c2f, 6 ), convertGlobalToLocalMap( cg2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); @@ -389,8 +333,6 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, // convert to cell local index CellLocIdx const & cli = g2l.cells.at( cgi ); - res.c2t[cli] = cellType; // may have been easier to do this with global indices - std::vector< FaceLocIdx > & faces = res.c2f[cli]; std::vector< EdgeLocIdx > & edges = res.c2e[cli]; @@ -511,12 +453,6 @@ UpwardMappings buildUpwardMappings( DownwardMappings const & downwardMappings ) } } - // Filling `t2c` - for( auto const & [cli, type]: downwardMappings.c2t ) - { - res.t2c[type].emplace_back( cli ); // may have been easier to do this with global indices - } - return res; } @@ -599,87 +535,11 @@ std::map< MpiRank, std::vector< LI > > invertGhostRecv( std::map< GI, MpiRank > return output; } -CellBlkImpl makeFlavorlessCellBlkImpls( std::size_t const & numCells, - std::map< CellLocIdx, std::vector< NodeLocIdx > > & c2n, - std::map< CellLocIdx, std::vector< EdgeLocIdx > > & c2e, - std::map< CellLocIdx, std::vector< FaceLocIdx > > & c2f, - std::map< CellGlbIdx, CellLocIdx > & cg2l, - std::map< CellGlbIdx, std::set< MpiRank > > & send, - std::map< CellGlbIdx, MpiRank > & recv, - std::map< geos::ElementType, std::vector< CellLocIdx > > & t2c, - std::map< CellLocIdx, CellGlbIdx > & cl2g ) -{ - GEOS_ASSERT_EQ( numCells, std::size( c2n ) ); - GEOS_ASSERT_EQ( numCells, std::size( c2e ) ); - GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); - GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); - - std::map< string, CellBlkImpl > ret; - - for (auto const & [cellType, clis]: t2c) - { - // if (cellType == ElementType::Wedge) {continue;} - // create structs that will be populated for this cell - string blkName = convertGeosxElementStringVtkStringGhosting(cellType); - std::size_t numCells_block = clis.size(); - std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n_block; - std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e_block; - std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f_block; - std::map< CellGlbIdx, CellLocIdx > cg2l_block; - std::map< CellGlbIdx, std::set< MpiRank > > send_block; - std::map< CellGlbIdx, MpiRank > recv_block; - - GEOS_LOG_RANK("building block of type " << cellType << " / " << blkName); - - // populate data structs for current cell type - for (auto const & cli: clis) - { - c2n_block[cli] = c2n[cli]; - c2e_block[cli] = c2e[cli]; - c2f_block[cli] = c2f[cli]; - //need the global index of this local index - cg2l_block[cl2g[cli]] = cli; - send_block[cl2g[cli]] = send[cl2g[cli]]; - recv_block[cl2g[cli]] = recv[cl2g[cli]]; - } - - // add name + CellBlkImpl (call constructor with new data) to ret map - // ret[blkName] = makeFlavorlessCellBlkImpl( cellType, - // numCells_block, - // c2n_block, - // c2e_block, - // c2f_block, - // cg2l_block, - // invertGhostSend(send_block, cg2l_block), - // invertGhostRecv(recv_block, cg2l_block) ); - - // ret[blkName] = makeFlavorlessCellBlkImpl( cellType, - // numCells, - // c2n, - // c2e, - // c2f, - // cg2l, - // invertGhostSend(send, cg2l), - // invertGhostRecv(recv, cg2l) ); - - // GEOS_LOG_RANK("Finished block of type " << cellType << " / " << blkName); - } - - return makeFlavorlessCellBlkImpl( ElementType::Hexahedron, - numCells, - c2n, - c2e, - c2f, - cg2l, - invertGhostSend(send, cg2l), - invertGhostRecv(recv, cg2l) ); -} - void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, - GhostRecv & recv, - GhostSend & send, + GhostRecv const & recv, + GhostSend const & send, MeshMappingImpl & meshMappings ) { // start by merging the 3 mesh graphs for the different types of data on the graph into 1 struct @@ -690,28 +550,20 @@ void buildPods( MeshGraph const & owned, // create GlobalToLocal, which maps global IDs to rank local IDs for (nodes, edges, faces, cells) // mapKeys is just a utility which extracts keys // buildGlobalToLocalMap just takes the global ids and maps them to 1, 2, 3, ... - GlobalToLocal g2l{ + GlobalToLocal const g2l{ buildGlobalToLocalMap( mapKeys< std::set >( graph.n2pos ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.e2n ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.f2e ) ), buildGlobalToLocalMap( mapKeys< std::set >( graph.c2f ) ) }; - // invert GLobal to local, as I needed one thing for multiple cell types - LocalToGlobal l2g{ - buildLocalToGlobalMap( g2l.nodes ), - buildLocalToGlobalMap( g2l.edges ), - buildLocalToGlobalMap( g2l.faces ), - buildLocalToGlobalMap( g2l.cells ) - }; - // Now we build the mappings used by GEOS // Downward - edges2nodes, faces2edges, faces2nodes, cells2faces, cells2edges, cells2nodes // The difference is that these use local indices, thus we built g2l // there are assumptions on what element types can be used here (to define c2n) - DownwardMappings downwardMappings = buildDownwardMappings( g2l, graph ); + DownwardMappings const downwardMappings = buildDownwardMappings( g2l, graph ); // invert to downward mappings to get e2f, f2c, n2e, n2f, n2c - UpwardMappings upwardMappings = buildUpwardMappings( downwardMappings ); + UpwardMappings const upwardMappings = buildUpwardMappings( downwardMappings ); GEOS_LOG_RANK("Built upward and downward mappings"); @@ -741,21 +593,13 @@ void buildPods( MeshGraph const & owned, g2l.nodes, invertGhostSend( send.nodes, g2l.nodes ), invertGhostRecv( recv.nodes, g2l.nodes ) ) ); - - - // here we need to make multiple cell blocks, instead of just one - // CellMgrImpl should accept multiple cell blocks, so that it can return them - // then the getCellBlocks should just return the member variable for CellMgr - // makeFlavorlessCellBlkImpl returns a CellBlkImpl, so a new version should return whatever CellMgrImpl wants - meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpls( std::size( g2l.cells ), + meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpl( std::size( g2l.cells ), downwardMappings.c2n, downwardMappings.c2e, downwardMappings.c2f, g2l.cells, - send.cells, - recv.cells, - upwardMappings.t2c, - l2g.cells ) ) ); + invertGhostSend( send.cells, g2l.cells ), + invertGhostRecv( recv.cells, g2l.cells ) ) ) ); meshMappings.setNeighbors( getNeighbors( recv, send ) ); diff --git a/src/coreComponents/mesh/generators/BuildPods.hpp b/src/coreComponents/mesh/generators/BuildPods.hpp index 6110fd7e1af..95315e95c51 100644 --- a/src/coreComponents/mesh/generators/BuildPods.hpp +++ b/src/coreComponents/mesh/generators/BuildPods.hpp @@ -88,8 +88,8 @@ struct GhostRecv void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, - GhostRecv & recv, - GhostSend & send, + GhostRecv const & recv, + GhostSend const & send, MeshMappingImpl & meshMappings ); } diff --git a/src/coreComponents/mesh/generators/NewGhosting.cpp b/src/coreComponents/mesh/generators/NewGhosting.cpp index 68ade942c39..533875406ea 100644 --- a/src/coreComponents/mesh/generators/NewGhosting.cpp +++ b/src/coreComponents/mesh/generators/NewGhosting.cpp @@ -59,7 +59,8 @@ ElementType convertVtkToGeosxElementTypeGhosting( vtkCell *cell ) case VTK_HEXAHEDRON: return ElementType::Hexahedron; default: { - GEOS_ERROR( cell->GetCellType() << " is not a recognized cell type in ghosting.\n" ); + GEOS_ERROR( cell->GetCellType() << " is not a recognized cell type in ghosting.\n" << + generalMeshErrorAdvice ); return {}; } } @@ -1460,7 +1461,7 @@ void doTheNewGhosting( vtkSmartPointer< vtkDataSet > mesh, // By doing some clever matrix products we can figure out which ranks need what automagically // `ghosts` is also a MeshGraph like owned and present, but described the geometric entities which are ghosted onto the rank from another owner // recv and send are of type GhostRecv and GhostSend (in BuildPods.hpp) and describe what (nodes, edges, faces, cells) each rank needs to receive from and send to others - auto [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); + auto const [ghosts, recv, send] = performGhosting( owned, present, matrixOffsets, curRank ); // Finally, we use everything we have to populate the mappings that interface with the rest of GEOS // Note that we have already done the ghosting, so these mappings are set once and for all diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index ada17439485..c9aa3f8ff34 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -91,8 +91,7 @@ ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const return { std::move( toBlockIndex ), m_f2c }; } -CellBlkImpl::CellBlkImpl( ElementType cellType, - localIndex numCells, +CellBlkImpl::CellBlkImpl( localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -103,44 +102,32 @@ CellBlkImpl::CellBlkImpl( ElementType cellType, m_numCells( numCells ), m_c2n( c2n ), m_c2e( c2e ), - m_c2f( c2f ), - m_cellType( cellType ) + m_c2f( c2f ) { } ElementType CellBlkImpl::getElementType() const { - // return m_cellType; return ElementType::Hexahedron; } localIndex CellBlkImpl::numNodesPerElement() const { - // return m_c2n.size(1); return 8; } localIndex CellBlkImpl::numEdgesPerElement() const { - // return m_c2e.size(1); return 12; } localIndex CellBlkImpl::numFacesPerElement() const { - // return m_c2f.size(1); return 6; } std::map< string, generators::CellBlk const * > CellMgrImpl::getCellBlks() const { - // std::map< string, generators::CellBlk const * > ret; - - // for (auto const & [blkName, cellBlk] : m_cellBlks) - // { - // ret[blkName] = & cellBlk; - // } - - return { { string( "hexahedra" ), &m_cellBlk } }; + return { { string( "hexahedra" ), &m_cellBlk } }; // TODO hard coded values. } } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index ab3b91ee5f0..6a22754d5b1 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -209,8 +209,7 @@ class CellBlkImpl : public generators::CellBlk public: CellBlkImpl() = default; - CellBlkImpl( ElementType cellType, - localIndex numCells, + CellBlkImpl( localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -267,7 +266,6 @@ class CellBlkImpl : public generators::CellBlk array2d< localIndex, cells::NODE_MAP_PERMUTATION > m_c2n; array2d< localIndex > m_c2e; array2d< localIndex > m_c2f; - ElementType m_cellType; }; class CellMgrImpl : public generators::CellMgr @@ -275,7 +273,6 @@ class CellMgrImpl : public generators::CellMgr public: CellMgrImpl() = default; - // CellMgrImpl( std::map< string, CellBlkImpl > && cellBlks ) CellMgrImpl( CellBlkImpl && cellBlks ) : m_cellBlk( cellBlks ) @@ -284,8 +281,7 @@ class CellMgrImpl : public generators::CellMgr [[nodiscard]] std::map< string, generators::CellBlk const * > getCellBlks() const override; private: - // std::map< string, CellBlkImpl > m_cellBlks; - CellBlkImpl m_cellBlk; + CellBlkImpl m_cellBlk; }; class MeshMappingImpl : public generators::MeshMappings From 1be9277c2b100f5df1705804596f575023b5de51 Mon Sep 17 00:00:00 2001 From: Ryan Aronson Date: Mon, 15 Jul 2024 18:48:01 +0000 Subject: [PATCH 106/106] Re-added impl of multi types which works for hex mesh, need to debug what I think is non-contiguous local indices --- src/coreComponents/mesh/ElementType.hpp | 60 +++++++++++++ .../mesh/generators/BuildPods.cpp | 87 +++++++++++++++++-- src/coreComponents/mesh/generators/Pods.cpp | 34 ++++++-- src/coreComponents/mesh/generators/Pods.hpp | 16 +++- 4 files changed, 179 insertions(+), 18 deletions(-) diff --git a/src/coreComponents/mesh/ElementType.hpp b/src/coreComponents/mesh/ElementType.hpp index a43f4328a37..b046934ca88 100644 --- a/src/coreComponents/mesh/ElementType.hpp +++ b/src/coreComponents/mesh/ElementType.hpp @@ -119,6 +119,66 @@ inline int getNumFaces3D( ElementType const elementType ) return -1; } +/** + * @brief Get number of edges for 3D elements + * @param elementType type of element + * @return number of edges for 3D elements (-1 if not a 3D element) + */ +inline int getNumEdges3D( ElementType const elementType ) +{ + switch( elementType ) + { + case ElementType::Vertex: + case ElementType::Line: + case ElementType::Triangle: + case ElementType::Quadrilateral: + case ElementType::Polygon: return -1; + case ElementType::Tetrahedron: return 6; + case ElementType::Pyramid: return 8; + case ElementType::Wedge: return 9; + case ElementType::Hexahedron: return 12; + case ElementType::Prism5: return -1; + case ElementType::Prism6: return -1; + case ElementType::Prism7: return -1; + case ElementType::Prism8: return -1; + case ElementType::Prism9: return -1; + case ElementType::Prism10: return -1; + case ElementType::Prism11: return -1; + case ElementType::Polyhedron: return -1; + } + return -1; +} + +/** + * @brief Get number of nodes for 3D elements + * @param elementType type of element + * @return number of modes for 3D elements (-1 if not a 3D element) + */ +inline int getNumNodes3D( ElementType const elementType ) +{ + switch( elementType ) + { + case ElementType::Vertex: + case ElementType::Line: + case ElementType::Triangle: + case ElementType::Quadrilateral: + case ElementType::Polygon: return -1; + case ElementType::Tetrahedron: return 4; + case ElementType::Pyramid: return 5; + case ElementType::Wedge: return 6; + case ElementType::Hexahedron: return 8; + case ElementType::Prism5: return -1; + case ElementType::Prism6: return -1; + case ElementType::Prism7: return -1; + case ElementType::Prism8: return -1; + case ElementType::Prism9: return -1; + case ElementType::Prism10: return -1; + case ElementType::Prism11: return -1; + case ElementType::Polyhedron: return -1; + } + return -1; +} + /// Strings for ElementType ENUM_STRINGS( ElementType, "Vertex", diff --git a/src/coreComponents/mesh/generators/BuildPods.cpp b/src/coreComponents/mesh/generators/BuildPods.cpp index 706c6f19eec..843b94cd39d 100644 --- a/src/coreComponents/mesh/generators/BuildPods.cpp +++ b/src/coreComponents/mesh/generators/BuildPods.cpp @@ -81,6 +81,7 @@ struct DownwardMappings std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n; std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f; std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e; + std::map< CellGlbIdx, ElementType > c2t; }; struct UpwardMappings @@ -90,6 +91,7 @@ struct UpwardMappings std::map< NodeLocIdx, std::vector< EdgeLocIdx > > n2e; std::map< NodeLocIdx, std::vector< FaceLocIdx > > n2f; std::map< NodeLocIdx, std::vector< CellLocIdx > > n2c; + std::map< ElementType, std::vector< CellGlbIdx > > t2c; }; // converts generic map to arrayOfArrays @@ -262,7 +264,8 @@ NodeMgrImpl makeFlavorlessNodeMgrImpl( std::size_t const & numNodes, toFlavorlessMapping( recv ) ); } -CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, +CellBlkImpl makeFlavorlessCellBlkImpl( ElementType cellType, + std::size_t const & numCells, std::map< CellLocIdx, std::vector< NodeLocIdx > > const & c2n, std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, @@ -275,10 +278,11 @@ CellBlkImpl makeFlavorlessCellBlkImpl( std::size_t const & numCells, GEOS_ASSERT_EQ( numCells, std::size( c2f ) ); GEOS_ASSERT_EQ( numCells, std::size( cg2l ) ); - return CellBlkImpl( intConv< localIndex >( numCells ), - convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, 8 ), - convertToA2d( c2e, 12 ), - convertToA2d( c2f, 6 ), + return CellBlkImpl( cellType, + intConv< localIndex >( numCells ), + convertToA2d< CellLocIdx, NodeLocIdx, cells::NODE_MAP_PERMUTATION >( c2n, getNumNodes3D(cellType) ), + convertToA2d( c2e, getNumEdges3D(cellType) ), + convertToA2d( c2f, getNumFaces3D(cellType) ), convertGlobalToLocalMap( cg2l ), toFlavorlessMapping( send ), toFlavorlessMapping( recv ) ); @@ -330,6 +334,8 @@ DownwardMappings buildDownwardMappings( GlobalToLocal const & g2l, geos::ElementType const & cellType = std::get<0>(typeAndFaces); std::vector const & faceInfos = std::get<1>(typeAndFaces); + res.c2t[cgi] = cellType; + // convert to cell local index CellLocIdx const & cli = g2l.cells.at( cgi ); @@ -453,6 +459,12 @@ UpwardMappings buildUpwardMappings( DownwardMappings const & downwardMappings ) } } + // Filling `t2c` + for( auto const & [cgi, cellType]: downwardMappings.c2t ) + { + res.t2c[cellType].emplace_back(cgi); + } + return res; } @@ -535,6 +547,63 @@ std::map< MpiRank, std::vector< LI > > invertGhostRecv( std::map< GI, MpiRank > return output; } +std::vector makeFlavorlessCellBlkImpls(std::size_t const & numCells, + std::map< CellLocIdx, std::vector< NodeLocIdx > > const & c2n, + std::map< CellLocIdx, std::vector< EdgeLocIdx > > const & c2e, + std::map< CellLocIdx, std::vector< FaceLocIdx > > const & c2f, + std::map< CellGlbIdx, CellLocIdx > const & cg2l, + std::map< CellGlbIdx, std::set< MpiRank > > const & send, + std::map< CellGlbIdx, MpiRank > const & recv, + std::map< CellGlbIdx, ElementType > const & c2t, + std::map< ElementType, std::vector< CellGlbIdx > > const & t2c ) +{ + + std::vector ret; + + for (auto & [cellType, cgis] : t2c) + { + + std::cout << "creating block of type " << cellType << std::endl; + + std::size_t numCells_blk = cgis.size(); + + std::map< CellLocIdx, std::vector< NodeLocIdx > > c2n_blk; + std::map< CellLocIdx, std::vector< EdgeLocIdx > > c2e_blk; + std::map< CellLocIdx, std::vector< FaceLocIdx > > c2f_blk; + std::map< CellGlbIdx, CellLocIdx > cg2l_blk; + std::map< CellGlbIdx, std::set< MpiRank > > send_blk; + std::map< CellGlbIdx, MpiRank > recv_blk; + + for (auto cgi : cgis) + { + CellLocIdx cli = cg2l.at(cgi); + c2n_blk[cli] = c2n.at(cli); + c2e_blk[cli] = c2e.at(cli); + c2f_blk[cli] = c2f.at(cli); + cg2l_blk[cgi] = cli; + if (send.count(cgi)) + { + send_blk[cgi] = send.at(cgi); + } + if (recv.count(cgi)) + { + recv_blk[cgi] = recv.at(cgi); + } + } + + ret.emplace_back( makeFlavorlessCellBlkImpl( cellType, + std::size( cg2l_blk ), + c2n_blk, + c2e_blk, + c2f_blk, + cg2l_blk, + invertGhostSend(send_blk, cg2l_blk), + invertGhostRecv(recv_blk, cg2l_blk) ) ); + } + + return ret; +} + void buildPods( MeshGraph const & owned, MeshGraph const & present, MeshGraph const & ghosts, @@ -593,13 +662,15 @@ void buildPods( MeshGraph const & owned, g2l.nodes, invertGhostSend( send.nodes, g2l.nodes ), invertGhostRecv( recv.nodes, g2l.nodes ) ) ); - meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpl( std::size( g2l.cells ), + meshMappings.setCellMgr( CellMgrImpl( makeFlavorlessCellBlkImpls(std::size( g2l.cells ), downwardMappings.c2n, downwardMappings.c2e, downwardMappings.c2f, g2l.cells, - invertGhostSend( send.cells, g2l.cells ), - invertGhostRecv( recv.cells, g2l.cells ) ) ) ); + send.cells, + recv.cells, + downwardMappings.c2t, + upwardMappings.t2c) ) ); meshMappings.setNeighbors( getNeighbors( recv, send ) ); diff --git a/src/coreComponents/mesh/generators/Pods.cpp b/src/coreComponents/mesh/generators/Pods.cpp index c9aa3f8ff34..884818f71ad 100644 --- a/src/coreComponents/mesh/generators/Pods.cpp +++ b/src/coreComponents/mesh/generators/Pods.cpp @@ -17,6 +17,20 @@ namespace geos { +std::string convertGeosxElementStringVtkStringGhosting( ElementType const & cellType) +{ + switch( cellType) + { + case ElementType::Hexahedron: return string("hexahedra"); + case ElementType::Wedge: return string("wedges"); + default: + { + GEOS_ERROR( cellType << " is not a recognized cell type in ghosting.\n" ); + return {}; + } + } +} + NodeMgrImpl::NodeMgrImpl( localIndex numNodes, array2d< real64, nodes::REFERENCE_POSITION_PERM > && positions, ArrayOfArrays< localIndex > const & n2e, @@ -91,7 +105,8 @@ ToCellRelation< array2d< localIndex > > FaceMgrImpl::getFaceToElements() const return { std::move( toBlockIndex ), m_f2c }; } -CellBlkImpl::CellBlkImpl( localIndex numCells, +CellBlkImpl::CellBlkImpl( ElementType cellType, + localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -99,6 +114,7 @@ CellBlkImpl::CellBlkImpl( localIndex numCells, std::map< integer, array1d< localIndex > > && send, std::map< integer, array1d< localIndex > > && recv ) : m_ghost{ std::move( g2l ), std::move( send ), std::move( recv ) }, + m_cellType( cellType ), m_numCells( numCells ), m_c2n( c2n ), m_c2e( c2e ), @@ -107,27 +123,33 @@ CellBlkImpl::CellBlkImpl( localIndex numCells, ElementType CellBlkImpl::getElementType() const { - return ElementType::Hexahedron; + return m_cellType; } localIndex CellBlkImpl::numNodesPerElement() const { - return 8; + return getNumNodes3D(m_cellType); } localIndex CellBlkImpl::numEdgesPerElement() const { - return 12; + return getNumEdges3D(m_cellType); } localIndex CellBlkImpl::numFacesPerElement() const { - return 6; + return getNumFaces3D(m_cellType); } std::map< string, generators::CellBlk const * > CellMgrImpl::getCellBlks() const { - return { { string( "hexahedra" ), &m_cellBlk } }; // TODO hard coded values. + std::map< string, generators::CellBlk const * > ret; + for (size_t i = 0; i < m_cellBlk.size(); ++i) + { + ret[convertGeosxElementStringVtkStringGhosting(m_cellBlk[i].getElementType())] = &m_cellBlk[i]; + } + + return ret; } } // end of namespace \ No newline at end of file diff --git a/src/coreComponents/mesh/generators/Pods.hpp b/src/coreComponents/mesh/generators/Pods.hpp index 6a22754d5b1..0ed563f3254 100644 --- a/src/coreComponents/mesh/generators/Pods.hpp +++ b/src/coreComponents/mesh/generators/Pods.hpp @@ -209,7 +209,8 @@ class CellBlkImpl : public generators::CellBlk public: CellBlkImpl() = default; - CellBlkImpl( localIndex numCells, + CellBlkImpl( ElementType cellType, + localIndex numCells, array2d< localIndex, cells::NODE_MAP_PERMUTATION > const & c2n, array2d< localIndex > const & c2e, array2d< localIndex > const & c2f, @@ -262,6 +263,7 @@ class CellBlkImpl : public generators::CellBlk private: GhostMapping m_ghost; // Diamond + ElementType m_cellType; localIndex m_numCells; array2d< localIndex, cells::NODE_MAP_PERMUTATION > m_c2n; array2d< localIndex > m_c2e; @@ -273,15 +275,21 @@ class CellMgrImpl : public generators::CellMgr public: CellMgrImpl() = default; - CellMgrImpl( CellBlkImpl && cellBlks ) + CellMgrImpl( std::vector< CellBlkImpl > && cellBlks ) : m_cellBlk( cellBlks ) - { } + { + std::cout << "CellMgr has blocks of:" << std::endl; + for (size_t i = 0; i < m_cellBlk.size(); ++i) + { + std::cout << m_cellBlk[i].getElementType() << std::endl; + } + } [[nodiscard]] std::map< string, generators::CellBlk const * > getCellBlks() const override; private: - CellBlkImpl m_cellBlk; + std::vector< CellBlkImpl > m_cellBlk; }; class MeshMappingImpl : public generators::MeshMappings