diff --git a/include/aero/AeroContainer.h b/include/aero/AeroContainer.h index 6021ab2bb..8d07a6c8b 100644 --- a/include/aero/AeroContainer.h +++ b/include/aero/AeroContainer.h @@ -55,6 +55,8 @@ class AeroContainer const stk::mesh::PartVector fsi_parts(); const stk::mesh::PartVector fsi_bndry_parts(); const std::vector fsi_bndry_part_names(); + double openfast_accumulated_time(); + double nalu_fsi_accumulated_time(); private: bool has_actuators() { return actuatorModel_.is_active(); } diff --git a/include/aero/fsi/OpenfastFSI.h b/include/aero/fsi/OpenfastFSI.h index 87fea294c..7e08fb685 100644 --- a/include/aero/fsi/OpenfastFSI.h +++ b/include/aero/fsi/OpenfastFSI.h @@ -47,6 +47,9 @@ class OpenfastFSI std::array axis, double omega, double curTime); void end_openfast(); + double total_openfastfsi_execution_time() { return openFastTimer_.second; } + double total_nalu_fsi_execution_time() { return naluTimer_.second; } + private: OpenfastFSI() = delete; OpenfastFSI(const OpenfastFSI&) = delete; @@ -58,6 +61,8 @@ class OpenfastFSI void compute_mapping(); void send_loads(const double curTime); + void timer_start(std::pair& timer); + void timer_stop(std::pair& timer); std::shared_ptr bulk_; @@ -79,6 +84,11 @@ class OpenfastFSI double dt_{-1.0}; // Store nalu-wind step + std::pair openFastTimer_{ + 0.0, 0.0}; // store time taken in openfast calls + std::pair naluTimer_{ + 0.0, 0.0}; // store time taken in openfast calls + int writeFreq_{ 30}; // Frequency to write line loads and deflections to netcdf file diff --git a/src/Realm.C b/src/Realm.C index b9327478c..69743afcb 100644 --- a/src/Realm.C +++ b/src/Realm.C @@ -4038,6 +4038,40 @@ Realm::dump_simulation_time() << std::endl; } + if (aeroModels_->has_fsi()) { + double naluFsiTimer = aeroModels_->nalu_fsi_accumulated_time(); + double openFastFsiTimer = aeroModels_->openfast_accumulated_time(); + // nalu fsi calculations + double g_totalNalu = 0.0, g_minNalu = 0.0, g_maxNalu = 0.0; + stk::all_reduce_min( + NaluEnv::self().parallel_comm(), &naluFsiTimer, &g_minNalu, 1); + stk::all_reduce_max( + NaluEnv::self().parallel_comm(), &naluFsiTimer, &g_maxNalu, 1); + stk::all_reduce_sum( + NaluEnv::self().parallel_comm(), &naluFsiTimer, &g_totalNalu, 1); + + NaluEnv::self().naluOutputP0() + << "Timing for FSI Computations : " << std::endl; + NaluEnv::self().naluOutputP0() + << " Nalu-Wind::computations -- " + << " \tavg: " << g_totalNalu / double(nprocs) << " \tmin: " << g_minNalu + << " \tmax: " << g_maxNalu << std::endl; + + // openfast calculations (excluding data fetch operations) + double g_totalFast = 0.0, g_minFast = 0.0, g_maxFast = 0.0; + stk::all_reduce_min( + NaluEnv::self().parallel_comm(), &openFastFsiTimer, &g_minFast, 1); + stk::all_reduce_max( + NaluEnv::self().parallel_comm(), &openFastFsiTimer, &g_maxFast, 1); + stk::all_reduce_sum( + NaluEnv::self().parallel_comm(), &openFastFsiTimer, &g_totalFast, 1); + + NaluEnv::self().naluOutputP0() + << " OpenFAST::computations -- " + << " \tavg: " << g_totalFast / double(nprocs) << " \tmin: " << g_minFast + << " \tmax: " << g_maxFast << std::endl; + } + // consolidated sort if (solutionOptions_->useConsolidatedSolverAlg_) { double g_totalSort = 0.0, g_minSort = 0.0, g_maxSort = 0.0; diff --git a/src/aero/AeroContainer.C b/src/aero/AeroContainer.C index 138fb5e10..ba53b87ba 100644 --- a/src/aero/AeroContainer.C +++ b/src/aero/AeroContainer.C @@ -213,5 +213,29 @@ AeroContainer::fsi_bndry_part_names() return bndry_part_names; } +double +AeroContainer::openfast_accumulated_time() +{ +#ifdef NALU_USES_OPENFAST_FSI + if (has_fsi()) + return fsiContainer_->total_openfastfsi_execution_time(); + else + return -1.0; +#endif + return -1.0; +} + +double +AeroContainer::nalu_fsi_accumulated_time() +{ +#ifdef NALU_USES_OPENFAST_FSI + if (has_fsi()) + return fsiContainer_->total_nalu_fsi_execution_time(); + else + return -1.0; +#endif + return -1.0; +} + } // namespace nalu } // namespace sierra diff --git a/src/aero/fsi/OpenfastFSI.C b/src/aero/fsi/OpenfastFSI.C index 535fb4a7c..9a6a882a4 100644 --- a/src/aero/fsi/OpenfastFSI.C +++ b/src/aero/fsi/OpenfastFSI.C @@ -382,21 +382,27 @@ OpenfastFSI::compute_mapping() void OpenfastFSI::predict_struct_states() { + timer_start(openFastTimer_); FAST.predict_states(); + timer_stop(openFastTimer_); } void OpenfastFSI::predict_struct_timestep(const double curTime) { send_loads(curTime); + timer_start(openFastTimer_); FAST.update_states_driver_time_step(); + timer_stop(openFastTimer_); } void OpenfastFSI::advance_struct_timestep(const double curTime) { + timer_start(openFastTimer_); FAST.advance_to_next_driver_time_step(); + timer_stop(openFastTimer_); tStep_ += 1; @@ -561,11 +567,13 @@ OpenfastFSI::compute_div_mesh_velocity() { int nTurbinesGlob = FAST.get_nTurbinesGlob(); + timer_start(naluTimer_); for (int i = 0; i < nTurbinesGlob; i++) { if (fsiTurbineData_[i] != NULL) // This may not be a turbine intended for // blade-resolved simulation fsiTurbineData_[i]->compute_div_mesh_velocity(); } + timer_stop(naluTimer_); } void @@ -584,6 +592,7 @@ void OpenfastFSI::map_displacements(double current_time, bool updateCurCoor) { + timer_start(naluTimer_); get_displacements(current_time); stk::mesh::Selector sel; @@ -627,12 +636,13 @@ OpenfastFSI::map_displacements(double current_time, bool updateCurCoor) curCoords->modify_on_host(); curCoords->sync_to_device(); } + timer_stop(naluTimer_); } void OpenfastFSI::map_loads(const int tStep, const double curTime) { - + timer_start(naluTimer_); int nTurbinesGlob = FAST.get_nTurbinesGlob(); for (int i = 0; i < nTurbinesGlob; i++) { if (fsiTurbineData_[i] != nullptr) { // This may not be a turbine intended @@ -661,6 +671,20 @@ OpenfastFSI::map_loads(const int tStep, const double curTime) fsiTurbineData_[i]->write_nc_def_loads(tStep, curTime); } } + timer_stop(naluTimer_); +} + +void +OpenfastFSI::timer_start(std::pair& timer) +{ + timer.first = NaluEnv::self().nalu_time(); +} + +void +OpenfastFSI::timer_stop(std::pair& timer) +{ + timer.first = NaluEnv::self().nalu_time() - timer.first; + timer.second += timer.first; } } // namespace nalu