From d74f655edad45ac242317c46d1544f9220b97692 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Tue, 2 Jul 2024 11:26:08 -0600 Subject: [PATCH 1/3] Disable unit test with 2 ranks as it can hang indefinitely when testing. (#1270) --- reg_tests/CTestList.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reg_tests/CTestList.cmake b/reg_tests/CTestList.cmake index 33b6afca59..2ba4247fd4 100644 --- a/reg_tests/CTestList.cmake +++ b/reg_tests/CTestList.cmake @@ -278,7 +278,7 @@ if(NOT ENABLE_CUDA AND NOT ENABLE_ROCM) # Unit tests #============================================================================= add_test_u(unitTest1 1) - add_test_u(unitTest2 2) + #add_test_u(unitTest2 2) # Can hang indefinitely #============================================================================= # Performance tests From 4faf299bb78c424661cc001240130713365d5305 Mon Sep 17 00:00:00 2001 From: Ilker Topcuoglu <114435459+itopcuoglu@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:40:13 -0400 Subject: [PATCH 2/3] Reverting PRs #1238 and #1240. Removing the set_resolutions option has made correcting iblanks at shared nodes obsolete, so the corrections are removed. (#1269) Co-authored-by: Jon Rood --- include/overset/TiogaBlock.h | 8 +------ src/overset/TiogaBlock.C | 42 +++--------------------------------- src/overset/TiogaSTKIface.C | 15 +++---------- 3 files changed, 7 insertions(+), 58 deletions(-) diff --git a/include/overset/TiogaBlock.h b/include/overset/TiogaBlock.h index 01d5be71cf..ddbed9170d 100644 --- a/include/overset/TiogaBlock.h +++ b/include/overset/TiogaBlock.h @@ -183,14 +183,8 @@ class TiogaBlock /** Update iblanks after connectivity updates */ - void update_iblanks(); - /** Update fringe and hole node vectors - */ - void update_fringe_and_hole_nodes( + void update_iblanks( std::vector&, std::vector&); - /** Update the Tioga view of iblanks prior to donor-to-receptor interpolation - */ - void update_tioga_iblanks(); /** Update element iblanks after connectivity updates */ diff --git a/src/overset/TiogaBlock.C b/src/overset/TiogaBlock.C index 3f89994528..7cc2c71825 100644 --- a/src/overset/TiogaBlock.C +++ b/src/overset/TiogaBlock.C @@ -224,7 +224,9 @@ TiogaBlock::update_connectivity() } void -TiogaBlock::update_iblanks() +TiogaBlock::update_iblanks( + std::vector& holeNodes, + std::vector& fringeNodes) { sierra::nalu::ScalarIntFieldType* ibf = meta_.get_field(stk::topology::NODE_RANK, "iblank"); @@ -239,25 +241,7 @@ TiogaBlock::update_iblanks() int* ib = stk::mesh::field_data(*ibf, *b); for (size_t in = 0; in < b->size(); in++) { ib[in] = ibnode(ip++); - } - } -} -void -TiogaBlock::update_fringe_and_hole_nodes( - std::vector& holeNodes, - std::vector& fringeNodes) -{ - sierra::nalu::ScalarIntFieldType* ibf = - meta_.get_field(stk::topology::NODE_RANK, "iblank"); - - stk::mesh::Selector mesh_selector = get_node_selector(blkParts_); - const stk::mesh::BucketVector& mbkts = - bulk_.get_buckets(stk::topology::NODE_RANK, mesh_selector); - - for (auto b : mbkts) { - int* ib = stk::mesh::field_data(*ibf, *b); - for (size_t in = 0; in < b->size(); in++) { if (ib[in] == 0) { holeNodes.push_back((*b)[in]); } else if (ib[in] == -1) { @@ -267,26 +251,6 @@ TiogaBlock::update_fringe_and_hole_nodes( } } -void -TiogaBlock::update_tioga_iblanks() -{ - sierra::nalu::ScalarIntFieldType* ibf = - meta_.get_field(stk::topology::NODE_RANK, "iblank"); - - stk::mesh::Selector mesh_selector = get_node_selector(blkParts_); - const stk::mesh::BucketVector& mbkts = - bulk_.get_buckets(stk::topology::NODE_RANK, mesh_selector); - - auto& ibnode = bdata_.iblank_.h_view; - int ip = 0; - for (auto b : mbkts) { - int* ib = stk::mesh::field_data(*ibf, *b); - for (size_t in = 0; in < b->size(); in++) { - ibnode(ip++) = ib[in]; - } - } -} - void TiogaBlock::update_iblank_cell() { diff --git a/src/overset/TiogaSTKIface.C b/src/overset/TiogaSTKIface.C index 5f65f53ab5..fac0ffe859 100644 --- a/src/overset/TiogaSTKIface.C +++ b/src/overset/TiogaSTKIface.C @@ -160,7 +160,8 @@ TiogaSTKIface::post_connectivity_work(const bool isDecoupled) { for (auto& tb : blocks_) { // Update IBLANK information at nodes and elements - tb->update_iblanks(); + tb->update_iblanks( + oversetManager_.holeNodes_, oversetManager_.fringeNodes_); tb->update_iblank_cell(); // For each block determine donor elements that needs to be ghosted to other @@ -173,17 +174,7 @@ TiogaSTKIface::post_connectivity_work(const bool isDecoupled) sierra::nalu::ScalarIntFieldType* ibf = meta_.get_field(stk::topology::NODE_RANK, "iblank"); std::vector pvec{ibf}; - stk::mesh::parallel_min(bulk_, {ibf}); - - for (auto& tb : blocks_) { - // Call update_iblanks again to assign holeNodes and fringeNodes vectors - // after iblanks on shared nodes are corrected - tb->update_fringe_and_hole_nodes( - oversetManager_.holeNodes_, oversetManager_.fringeNodes_); - // Return the corrected iblank field to Tioga prior to donor-to-receptor - // interpolation - tb->update_tioga_iblanks(); - } + stk::mesh::copy_owned_to_shared(bulk_, pvec); post_connectivity_sync(); From 53f493954891937db39876452af46dbf3ce3811c Mon Sep 17 00:00:00 2001 From: psakievich Date: Wed, 17 Jul 2024 11:50:03 -0600 Subject: [PATCH 3/3] CI to use exawind-manager containers (#1271) * CI to use exawind-manager containers --- .github/workflows/ci.yml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e37789afc..c0038f6f66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: container: image: ecpe4s/exawind-snapshot env: - SPACK_MANAGER: /spack-manager + EXAWIND_MANAGER: /exawind-manager E4S_MACHINE: true steps: - name: Cancel previous runs @@ -38,13 +38,16 @@ jobs: - name: Tests run: | /bin/bash -c " \ - source ${SPACK_MANAGER}/start.sh && \ - mkdir -p ${SPACK_MANAGER}/environments/exawind && \ - cd ${SPACK_MANAGER}/environments/exawind && \ + source ${EXAWIND_MANAGER}/start.sh && \ + mkdir -p ${EXAWIND_MANAGER}/environments/exawind && \ + cd ${EXAWIND_MANAGER}/environments/exawind && \ ln -s ${GITHUB_WORKSPACE} nalu-wind && \ - source ${SPACK_MANAGER}/start.sh && \ + spack-start && \ + spack -e e4s-build config add view:true && \ + spack -e e4s-build add cmake && \ + spack -e e4s-build install && \ quick-create-dev -s nalu-wind@master+tioga+hypre && \ - spack manager external --latest --blacklist nalu-wind gmake && \ + spack manager external ${EXAWIND_MANAGER}/environments/e4s-build --exclude nalu-wind gmake && \ spack install && \ spack cd -b nalu-wind && \ spack build-env nalu-wind ctest -j $(nproc) -L unit --output-on-failure \ @@ -55,7 +58,7 @@ jobs: container: image: ecpe4s/exawind-snapshot env: - SPACK_MANAGER: /spack-manager + EXAWIND_MANAGER: /exawind-manager E4S_MACHINE: true steps: - name: Cancel previous runs @@ -69,13 +72,16 @@ jobs: - name: Tests run: | /bin/bash -c " \ - source ${SPACK_MANAGER}/start.sh && \ - mkdir -p ${SPACK_MANAGER}/environments/exawind && \ - cd ${SPACK_MANAGER}/environments/exawind && \ + source ${EXAWIND_MANAGER}/start.sh && \ + mkdir -p ${EXAWIND_MANAGER}/environments/exawind && \ + cd ${EXAWIND_MANAGER}/environments/exawind && \ ln -s ${GITHUB_WORKSPACE} nalu-wind && \ - source ${SPACK_MANAGER}/start.sh && \ + spack-start && \ + spack -e e4s-build config add view:true && \ + spack -e e4s-build add cmake && \ + spack -e e4s-build install && \ quick-create-dev -s nalu-wind@master+tioga+hypre~trilinos-solvers && \ - spack manager external --latest --blacklist nalu-wind gmake && \ + spack manager external ${EXAWIND_MANAGER}/environments/e4s-build --exclude nalu-wind gmake && \ spack install && \ spack cd -b nalu-wind && \ spack build-env nalu-wind ctest -j $(nproc) -L unit --output-on-failure \