Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: IO timers #3480

Merged
merged 9 commits into from
Dec 7, 2024
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 50 additions & 34 deletions src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,53 +129,57 @@
}

///////////////////////////////////////////////////////////////////////////////////////////////////
bool BlueprintOutput::execute( real64 const time,
real64 const,
integer const cycle,
integer const,
real64 const,
bool BlueprintOutput::execute( real64 const time_n,

Check warning on line 132 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L132

Added line #L132 was not covered by tests
real64 const GEOS_UNUSED_PARAM( dt ),
integer const cycleNumber,
integer const GEOS_UNUSED_PARAM( eventCounter ),
real64 const GEOS_UNUSED_PARAM( eventProgress ),
DomainPartition & domain )
{
GEOS_MARK_FUNCTION;

MeshLevel const & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization();
{
Timer timer( m_outputTimer );

Check warning on line 142 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L142

Added line #L142 was not covered by tests

conduit::Node meshRoot;
conduit::Node & mesh = meshRoot[ "mesh" ];
conduit::Node & coordset = mesh[ "coordsets/nodes" ];
conduit::Node & topologies = mesh[ "topologies" ];
MeshLevel const & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization();

Check warning on line 144 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L144

Added line #L144 was not covered by tests

mesh[ "state/time" ] = time;
mesh[ "state/cycle" ] = cycle;
conduit::Node meshRoot;
conduit::Node & mesh = meshRoot[ "mesh" ];
conduit::Node & coordset = mesh[ "coordsets/nodes" ];
conduit::Node & topologies = mesh[ "topologies" ];

Check warning on line 149 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L146-L149

Added lines #L146 - L149 were not covered by tests

addNodalData( meshLevel.getNodeManager(), coordset, topologies, mesh[ "fields" ] );
mesh[ "state/time" ] = time_n;
mesh[ "state/cycle" ] = cycleNumber;

Check warning on line 152 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L151-L152

Added lines #L151 - L152 were not covered by tests

dataRepository::Group averagedElementData( "averagedElementData", this );
addElementData( meshLevel.getElemManager(), coordset, topologies, mesh[ "fields" ], averagedElementData );
addNodalData( meshLevel.getNodeManager(), coordset, topologies, mesh[ "fields" ] );

Check warning on line 154 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L154

Added line #L154 was not covered by tests

/// The Blueprint will complain if the fields node is present but empty.
if( mesh[ "fields" ].number_of_children() == 0 )
{
mesh.remove( "fields" );
}
dataRepository::Group averagedElementData( "averagedElementData", this );
addElementData( meshLevel.getElemManager(), coordset, topologies, mesh[ "fields" ], averagedElementData );

Check warning on line 157 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L156-L157

Added lines #L156 - L157 were not covered by tests

/// The Blueprint will complain if the fields node is present but empty.
if( mesh[ "fields" ].number_of_children() == 0 )

Check warning on line 160 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L160

Added line #L160 was not covered by tests
{
mesh.remove( "fields" );

Check warning on line 162 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L162

Added line #L162 was not covered by tests
}

/// Verify that the mesh conforms to the Blueprint.
conduit::Node info;
GEOS_ASSERT_MSG( conduit::blueprint::verify( "mesh", meshRoot, info ), info.to_json() );
/// Verify that the mesh conforms to the Blueprint.
conduit::Node info;
GEOS_ASSERT_MSG( conduit::blueprint::verify( "mesh", meshRoot, info ), info.to_json() );

Check warning on line 167 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L166-L167

Added lines #L166 - L167 were not covered by tests

/// Generate the Blueprint index.
conduit::Node fileRoot;
conduit::Node & index = fileRoot[ "blueprint_index/mesh" ];
conduit::blueprint::mesh::generate_index( mesh, "mesh", MpiWrapper::commSize(), index );
/// Generate the Blueprint index.
conduit::Node fileRoot;
conduit::Node & index = fileRoot[ "blueprint_index/mesh" ];
conduit::blueprint::mesh::generate_index( mesh, "mesh", MpiWrapper::commSize(), index );

Check warning on line 172 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L170-L172

Added lines #L170 - L172 were not covered by tests

/// Verify that the index conforms to the Blueprint.
info.reset();
GEOS_ASSERT_MSG( conduit::blueprint::mesh::index::verify( index, info ), info.to_json() );
/// Verify that the index conforms to the Blueprint.
info.reset();
GEOS_ASSERT_MSG( conduit::blueprint::mesh::index::verify( index, info ), info.to_json() );

Check warning on line 176 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L175-L176

Added lines #L175 - L176 were not covered by tests

/// Write out the root index file, then write out the mesh.
string const completePath = GEOS_FMT( "{}/blueprintFiles/cycle_{:07}", OutputBase::getOutputDirectory(), cycle );
string const filePathForRank = dataRepository::writeRootFile( fileRoot, completePath );
conduit::relay::io::save( meshRoot, filePathForRank, "hdf5" );
/// Write out the root index file, then write out the mesh.
string const completePath = GEOS_FMT( "{}/blueprintFiles/cycle_{:07}", OutputBase::getOutputDirectory(), cycleNumber );
string const filePathForRank = dataRepository::writeRootFile( fileRoot, completePath );
conduit::relay::io::save( meshRoot, filePathForRank, "hdf5" );
}

Check warning on line 182 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L179-L182

Added lines #L179 - L182 were not covered by tests

return false;
}
Expand Down Expand Up @@ -307,7 +311,19 @@
} );
}

namespace logInfo
{
struct BlueprintOutputTimer : public OutputTimerBase
{
std::string_view getDescription() const override { return "Blueprint output timing"; }

Check warning on line 318 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L318

Added line #L318 was not covered by tests
};
}

logInfo::OutputTimerBase const & BlueprintOutput::getTimerCategory() const

Check warning on line 322 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L322

Added line #L322 was not covered by tests
{
static logInfo::BlueprintOutputTimer timer;
return timer;

Check warning on line 325 in src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp#L325

Added line #L325 was not covered by tests
}

REGISTER_CATALOG_ENTRY( OutputBase, BlueprintOutput, string const &, dataRepository::Group * const )

Expand Down
6 changes: 6 additions & 0 deletions src/coreComponents/fileIO/Outputs/BlueprintOutput.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ class ElementRegionManager;
*/
class BlueprintOutput : public OutputBase
{
protected:
/**
* @copydoc OutputBase::getTimerCategory
*/
logInfo::OutputTimerBase const & getTimerCategory() const override;

public:

/**
Expand Down
14 changes: 14 additions & 0 deletions src/coreComponents/fileIO/Outputs/ChomboIO.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,20 @@

using namespace dataRepository;

namespace logInfo
{
struct ChomboOutputTimer : public OutputTimerBase
{
std::string_view getDescription() const override { return "Chombo output timing"; }

Check warning on line 37 in src/coreComponents/fileIO/Outputs/ChomboIO.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/ChomboIO.cpp#L37

Added line #L37 was not covered by tests
};
}

logInfo::OutputTimerBase const & ChomboIO::getTimerCategory() const

Check warning on line 41 in src/coreComponents/fileIO/Outputs/ChomboIO.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/ChomboIO.cpp#L41

Added line #L41 was not covered by tests
{
static logInfo::ChomboOutputTimer timer;
return timer;

Check warning on line 44 in src/coreComponents/fileIO/Outputs/ChomboIO.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/ChomboIO.cpp#L44

Added line #L44 was not covered by tests
}

ChomboIO::ChomboIO( string const & name, Group * const parent ):
OutputBase( name, parent ),
m_coupler( nullptr ),
Expand Down
6 changes: 6 additions & 0 deletions src/coreComponents/fileIO/Outputs/ChomboIO.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@ class ChomboIO final : public OutputBase
} viewKeys;
/// @endcond

protected:
/**
* @copydoc OutputBase::getTimerCategory
*/
logInfo::OutputTimerBase const & getTimerCategory() const override;

private:
ChomboCoupler * m_coupler;
string m_outputPath;
Expand Down
21 changes: 21 additions & 0 deletions src/coreComponents/fileIO/Outputs/OutputBase.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
OutputBase::OutputBase( string const & name,
Group * const parent ):
ExecutableGroup( name, parent ),
m_outputTimer(),
m_childDirectory(),
m_parallelThreads( 1 )
{
Expand All @@ -43,6 +44,8 @@
setInputFlag( InputFlags::OPTIONAL ).
setDescription( "Number of plot files." );

// Add the Timers log level
addLogLevel< logInfo::OutputTimers >();
}

OutputBase::~OutputBase()
Expand Down Expand Up @@ -106,5 +109,23 @@
}
}

void OutputBase::cleanup( real64 const GEOS_UNUSED_PARAM( time_n ),

Check warning on line 112 in src/coreComponents/fileIO/Outputs/OutputBase.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/OutputBase.cpp#L112

Added line #L112 was not covered by tests
integer const GEOS_UNUSED_PARAM( cycleNumber ),
integer const GEOS_UNUSED_PARAM( eventCounter ),
real64 const GEOS_UNUSED_PARAM( eventProgress ),
DomainPartition & GEOS_UNUSED_PARAM( domain ) )
{
// Report timing statistics
real64 const time = std::chrono::duration< double >( m_outputTimer ).count();
real64 const minTime = MpiWrapper::min( time );
real64 const maxTime = MpiWrapper::max( time );
if( maxTime > 0 )

Check warning on line 122 in src/coreComponents/fileIO/Outputs/OutputBase.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/OutputBase.cpp#L119-L122

Added lines #L119 - L122 were not covered by tests
{
GEOS_LOG_LEVEL_INFO_RANK_0( logInfo::OutputTimers,

Check warning on line 124 in src/coreComponents/fileIO/Outputs/OutputBase.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/OutputBase.cpp#L124

Added line #L124 was not covered by tests
GEOS_FMT( "{}: file writing time = {} s (min), {} s (max)",
getName(), minTime, maxTime ) );
}
}

Check warning on line 128 in src/coreComponents/fileIO/Outputs/OutputBase.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/OutputBase.cpp#L128

Added line #L128 was not covered by tests


} /* namespace geos */
57 changes: 56 additions & 1 deletion src/coreComponents/fileIO/Outputs/OutputBase.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,50 @@

#include "dataRepository/Group.hpp"
#include "dataRepository/ExecutableGroup.hpp"

#include "dataRepository/LogLevelsInfo.hpp" // For logInfo namespace
#include "common/Timer.hpp"

namespace geos
{

namespace logInfo
{
/**
* @brief Base timer category for output operations
* @details Provides configuration for logging output operation timing information
*/
struct OutputTimers
{
/**
* @brief Get the description of this timer
* @return String view containing the timer description
*/
static std::string_view getDescription() { return "Output timing information"; }

/**
* @brief Get the minimum log level for this timer
* @return Integer representing the minimum log level
*/
static constexpr int getMinLogLevel() { return 1; }
};

/**
* @brief Base interface for specific output type timers
* @details Each output type (VTK, Silo, etc.) implements this interface to provide
* its own timing category. This is used in conjunction with OutputTimers:
* - OutputTimerBase: For polymorphic behavior in derived output classes
* - OutputTimers: For the general output timing logging infrastructure
*/
struct OutputTimerBase
{
/**
* @brief Get the description of this timer
* @return String view containing the timer description
*/
virtual std::string_view getDescription() const = 0;
};
}

/**
* @class OutputBase
*
Expand Down Expand Up @@ -102,6 +141,22 @@ class OutputBase : public ExecutableGroup
**/
virtual void initializePreSubGroups() override;

/// Timer used to track duration of file writing operations for this specific output type
std::chrono::system_clock::duration m_outputTimer;

/**
* @brief Get the timer category for this output type
* @return Reference to the output timer base for timing statistics
*/
virtual logInfo::OutputTimerBase const & getTimerCategory() const = 0;

/// @copydoc geos::ExecutableGroup::cleanup
virtual void cleanup( real64 const time_n,
integer const cycleNumber,
integer const eventCounter,
real64 const eventProgress,
DomainPartition & domain ) override;

private:
string m_childDirectory;
integer m_parallelThreads;
Expand Down
14 changes: 14 additions & 0 deletions src/coreComponents/fileIO/Outputs/PythonOutput.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,20 @@
namespace geos
{

namespace logInfo
{
struct PythonOutputTimer : public OutputTimerBase
{
std::string_view getDescription() const override { return "Python output timing"; }

Check warning on line 25 in src/coreComponents/fileIO/Outputs/PythonOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/PythonOutput.cpp#L25

Added line #L25 was not covered by tests
};
}

logInfo::OutputTimerBase const & PythonOutput::getTimerCategory() const

Check warning on line 29 in src/coreComponents/fileIO/Outputs/PythonOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/PythonOutput.cpp#L29

Added line #L29 was not covered by tests
{
static logInfo::PythonOutputTimer timer;
return timer;

Check warning on line 32 in src/coreComponents/fileIO/Outputs/PythonOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/PythonOutput.cpp#L32

Added line #L32 was not covered by tests
}

REGISTER_CATALOG_ENTRY( OutputBase, PythonOutput, string const &, dataRepository::Group * const )

} // namespace geos
3 changes: 3 additions & 0 deletions src/coreComponents/fileIO/Outputs/PythonOutput.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ class PythonOutput : public OutputBase
GEOS_UNUSED_VAR( domain );
return true;
}

protected:
logInfo::OutputTimerBase const & getTimerCategory() const override;
};


Expand Down
29 changes: 21 additions & 8 deletions src/coreComponents/fileIO/Outputs/RestartOutput.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@

using namespace dataRepository;

namespace logInfo
{
struct RestartOutputTimer : public OutputTimerBase
{
std::string_view getDescription() const override { return "Restart output timing"; }

Check warning on line 31 in src/coreComponents/fileIO/Outputs/RestartOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/RestartOutput.cpp#L31

Added line #L31 was not covered by tests
};
}

RestartOutput::RestartOutput( string const & name,
Group * const parent ):
OutputBase( name, parent )
Expand All @@ -41,19 +49,24 @@
{
GEOS_MARK_FUNCTION;

Group & rootGroup = this->getGroupByPath( "/Problem" );
{
Timer timer( m_outputTimer );

Check warning on line 53 in src/coreComponents/fileIO/Outputs/RestartOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/RestartOutput.cpp#L53

Added line #L53 was not covered by tests

// Ignoring the eventProgress indicator for now to be compliant with the integrated test repo
// integer const eventProgressPercent = static_cast<integer const>(eventProgress * 100.0);
string const fileName = GEOS_FMT( "{}_restart_{:09}", getFileNameRoot(), cycleNumber );

rootGroup.prepareToWrite();
writeTree( joinPath( OutputBase::getOutputDirectory(), fileName ), *(rootGroup.getConduitNode().parent()) );
rootGroup.finishWriting();
Group & rootGroup = this->getGroupByPath( "/Problem" );
string const fileName = GEOS_FMT( "{}_restart_{:09}", getFileNameRoot(), cycleNumber );
rootGroup.prepareToWrite();
writeTree( joinPath( OutputBase::getOutputDirectory(), fileName ), *(rootGroup.getConduitNode().parent()) );
rootGroup.finishWriting();
}

Check warning on line 60 in src/coreComponents/fileIO/Outputs/RestartOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/RestartOutput.cpp#L55-L60

Added lines #L55 - L60 were not covered by tests

return false;
}

logInfo::OutputTimerBase const & RestartOutput::getTimerCategory() const

Check warning on line 65 in src/coreComponents/fileIO/Outputs/RestartOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/RestartOutput.cpp#L65

Added line #L65 was not covered by tests
{
static logInfo::RestartOutputTimer timer;
return timer;

Check warning on line 68 in src/coreComponents/fileIO/Outputs/RestartOutput.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/fileIO/Outputs/RestartOutput.cpp#L68

Added line #L68 was not covered by tests
}

REGISTER_CATALOG_ENTRY( OutputBase, RestartOutput, string const &, Group * const )
} /* namespace geos */
6 changes: 6 additions & 0 deletions src/coreComponents/fileIO/Outputs/RestartOutput.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ class RestartOutput : public OutputBase
dataRepository::ViewKey writeFEMFaces = { "writeFEMFaces" };
} viewKeys;
/// @endcond

protected:
/**
* @copydoc OutputBase::getTimerCategory
*/
logInfo::OutputTimerBase const & getTimerCategory() const override;
};


Expand Down
Loading
Loading