Merge branch 'master' into f/fm-kernel-ex

Exawind · Apr 1, 2024 · 0d0eceb · 0d0eceb
2 parents a576a5b + 30a8102
commit 0d0eceb
Show file tree

Hide file tree

Showing 6 changed files with 65 additions and 13 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -56,10 +56,6 @@ endif()
 
 add_library(nalu "")
 
-########################## MPI ####################################
-find_package(MPI REQUIRED)
-target_link_libraries(nalu PUBLIC $<$<BOOL:${MPI_CXX_FOUND}>:MPI::MPI_CXX>)
-
 if(ENABLE_CUDA)
   enable_language(CUDA)
   find_package(CUDAToolkit REQUIRED)
@@ -185,6 +181,7 @@ if(ENABLE_OPENFAST_FSI)
 endif()
 if(ENABLE_OPENFAST)
   set(CMAKE_PREFIX_PATH ${OpenFAST_DIR} ${CMAKE_PREFIX_PATH})
+  enable_language(Fortran)
   find_package(OpenFAST QUIET REQUIRED)
   message(STATUS "Found OpenFAST = ${OpenFAST_LIBRARY_DIRS}")
   target_link_libraries(nalu PUBLIC ${OpenFAST_LIBRARIES} ${OpenFAST_CPP_LIBRARIES})
@@ -205,6 +202,10 @@ if(ENABLE_TIOGA)
   endif()
 endif()
 
+########################## MPI ####################################
+find_package(MPI REQUIRED)
+target_link_libraries(nalu PUBLIC $<$<BOOL:${MPI_CXX_FOUND}>:MPI::MPI_CXX>)
+target_link_libraries(nalu PUBLIC $<$<BOOL:${MPI_Fortran_FOUND}>:MPI::MPI_Fortran>)
 
 ############################ MATRIXREE #####################################
 if(ENABLE_MATRIXFREE)

diff --git a/include/overset/TiogaBlock.h b/include/overset/TiogaBlock.h
@@ -178,8 +178,14 @@ class TiogaBlock
 
   /** Update iblanks after connectivity updates
    */
-  void update_iblanks(
+  void update_iblanks();
+  /** Update fringe and hole node vectors
+   */
+  void update_fringe_and_hole_nodes(
     std::vector<stk::mesh::Entity>&, std::vector<stk::mesh::Entity>&);
+  /** Update the Tioga view of iblanks prior to donor-to-receptor interpolation
+   */
+  void update_tioga_iblanks();
 
   /** Update element iblanks after connectivity updates
    */

diff --git a/src/Realm.C b/src/Realm.C
@@ -3416,10 +3416,10 @@ Realm::populate_restart(double& timeStepNm1, int& timeStepCount)
         auto* field = stk::mesh::get_field_by_name(fname, meta_data());
         if (field == nullptr)
           continue;
-
         const unsigned numStates = field->number_of_states();
         for (unsigned i = 0; i < numStates; ++i) {
           auto* fld = field->field_state(static_cast<stk::mesh::FieldState>(i));
+          fld->clear_sync_state();
           fld->modify_on_host();
           ngp_field_manager().get_field<double>(fld->mesh_meta_data_ordinal());
           fld->sync_to_device();

diff --git a/src/WallDistEquationSystem.C b/src/WallDistEquationSystem.C
@@ -68,7 +68,7 @@ namespace nalu {
 
 WallDistEquationSystem::WallDistEquationSystem(EquationSystems& eqSystems)
   : EquationSystem(eqSystems, "WallDistEQS", "ndtw"),
-    nodalGradAlgDriver_(realm_, "ndtw", "dwalldistdx"),
+    nodalGradAlgDriver_(realm_, "wall_distance_phi", "dwalldistdx"),
     managePNG_(realm_.get_consistent_mass_matrix_png("ndtw"))
 {
   if (managePNG_)

diff --git a/src/overset/TiogaBlock.C b/src/overset/TiogaBlock.C
@@ -224,9 +224,7 @@ TiogaBlock::update_connectivity()
 }
 
 void
-TiogaBlock::update_iblanks(
-  std::vector<stk::mesh::Entity>& holeNodes,
-  std::vector<stk::mesh::Entity>& fringeNodes)
+TiogaBlock::update_iblanks()
 {
   sierra::nalu::ScalarIntFieldType* ibf =
     meta_.get_field<int>(stk::topology::NODE_RANK, "iblank");
@@ -241,7 +239,25 @@ TiogaBlock::update_iblanks(
     int* ib = stk::mesh::field_data(*ibf, *b);
     for (size_t in = 0; in < b->size(); in++) {
       ib[in] = ibnode(ip++);
+    }
+  }
+}
 
+void
+TiogaBlock::update_fringe_and_hole_nodes(
+  std::vector<stk::mesh::Entity>& holeNodes,
+  std::vector<stk::mesh::Entity>& fringeNodes)
+{
+  sierra::nalu::ScalarIntFieldType* ibf =
+    meta_.get_field<int>(stk::topology::NODE_RANK, "iblank");
+
+  stk::mesh::Selector mesh_selector = get_node_selector(blkParts_);
+  const stk::mesh::BucketVector& mbkts =
+    bulk_.get_buckets(stk::topology::NODE_RANK, mesh_selector);
+
+  for (auto b : mbkts) {
+    int* ib = stk::mesh::field_data(*ibf, *b);
+    for (size_t in = 0; in < b->size(); in++) {
       if (ib[in] == 0) {
         holeNodes.push_back((*b)[in]);
       } else if (ib[in] == -1) {
@@ -251,6 +267,26 @@ TiogaBlock::update_iblanks(
   }
 }
 
+void
+TiogaBlock::update_tioga_iblanks()
+{
+  sierra::nalu::ScalarIntFieldType* ibf =
+    meta_.get_field<int>(stk::topology::NODE_RANK, "iblank");
+
+  stk::mesh::Selector mesh_selector = get_node_selector(blkParts_);
+  const stk::mesh::BucketVector& mbkts =
+    bulk_.get_buckets(stk::topology::NODE_RANK, mesh_selector);
+
+  auto& ibnode = bdata_.iblank_.h_view;
+  int ip = 0;
+  for (auto b : mbkts) {
+    int* ib = stk::mesh::field_data(*ibf, *b);
+    for (size_t in = 0; in < b->size(); in++) {
+      ibnode(ip++) = ib[in];
+    }
+  }
+}
+
 void
 TiogaBlock::update_iblank_cell()
 {

diff --git a/src/overset/TiogaSTKIface.C b/src/overset/TiogaSTKIface.C
@@ -164,8 +164,7 @@ TiogaSTKIface::post_connectivity_work(const bool isDecoupled)
 {
   for (auto& tb : blocks_) {
     // Update IBLANK information at nodes and elements
-    tb->update_iblanks(
-      oversetManager_.holeNodes_, oversetManager_.fringeNodes_);
+    tb->update_iblanks();
     tb->update_iblank_cell();
 
     // For each block determine donor elements that needs to be ghosted to other
@@ -178,7 +177,17 @@ TiogaSTKIface::post_connectivity_work(const bool isDecoupled)
   sierra::nalu::ScalarIntFieldType* ibf =
     meta_.get_field<int>(stk::topology::NODE_RANK, "iblank");
   std::vector<const stk::mesh::FieldBase*> pvec{ibf};
-  stk::mesh::copy_owned_to_shared(bulk_, pvec);
+  stk::mesh::parallel_min(bulk_, {ibf});
+
+  for (auto& tb : blocks_) {
+    // Call update_iblanks again to assign holeNodes and fringeNodes vectors
+    // after iblanks on shared nodes are corrected
+    tb->update_fringe_and_hole_nodes(
+      oversetManager_.holeNodes_, oversetManager_.fringeNodes_);
+    // Return the corrected iblank field to Tioga prior to donor-to-receptor
+    // interpolation
+    tb->update_tioga_iblanks();
+  }
 
   post_connectivity_sync();