diff --git a/.integrated_tests.yaml b/.integrated_tests.yaml index 0a3900f23a9..7aad59dc92f 100644 --- a/.integrated_tests.yaml +++ b/.integrated_tests.yaml @@ -1,6 +1,6 @@ baselines: bucket: geosx - baseline: integratedTests/baseline_integratedTests-pr3439-8919-b09db43 + baseline: integratedTests/baseline_integratedTests-pr2637-8940-3c0fc5c allow_fail: all: '' streak: '' diff --git a/BASELINE_NOTES.md b/BASELINE_NOTES.md index 8e9abcd6c27..15417af0b13 100644 --- a/BASELINE_NOTES.md +++ b/BASELINE_NOTES.md @@ -6,6 +6,10 @@ This file is designed to track changes to the integrated test baselines. Any developer who updates the baseline ID in the .integrated_tests.yaml file is expected to create an entry in this file with the pull request number, date, and their justification for rebaselining. These notes should be in reverse-chronological order, and use the following time format: (YYYY-MM-DD). +PR #2637 (2024-11-21) +===================== +Added numberOfTargetProcesses. + PR #3439 (2024-11-20) ===================== EDFM bugfixes: derivatives sign, frac/cell element volume, fix apertures inconsistency in test cases. diff --git a/src/coreComponents/fileIO/Outputs/VTKOutput.cpp b/src/coreComponents/fileIO/Outputs/VTKOutput.cpp index da364d0e4db..15e4b1dac13 100644 --- a/src/coreComponents/fileIO/Outputs/VTKOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/VTKOutput.cpp @@ -18,7 +18,7 @@ */ #include "VTKOutput.hpp" - +#include "common/MpiWrapper.hpp" #if defined(GEOS_USE_PYGEOSX) #include "fileIO/python/PyVTKOutputType.hpp" @@ -56,6 +56,11 @@ VTKOutput::VTKOutput( string const & name, setInputFlag( InputFlags::OPTIONAL ). setDescription( "Level detail plot. Only fields with lower of equal plot level will be output." ); + registerWrapper( viewKeysStruct::numberOfTargetProcesses, &m_numberOfTargetProcesses ). + setApplyDefaultValue( MpiWrapper::commSize() ). + setInputFlag( InputFlags::OPTIONAL ). + setDescription( "Number of output aggregate files to be written." ); + registerWrapper( viewKeysStruct::writeGhostCells, &m_writeGhostCells ). setApplyDefaultValue( 0 ). setInputFlag( InputFlags::OPTIONAL ). @@ -102,6 +107,14 @@ void VTKOutput::postInputInitialization() m_writer.setLevelNames( m_levelNames.toViewConst() ); m_writer.setOnlyPlotSpecifiedFieldNamesFlag( m_onlyPlotSpecifiedFieldNames ); + GEOS_ERROR_IF_LT_MSG( m_numberOfTargetProcesses, 1, + GEOS_FMT( "{}: processes count cannot be less than 1.", + getWrapperDataContext( viewKeysStruct::numberOfTargetProcesses ) ) ); + GEOS_ERROR_IF_GT_MSG( m_numberOfTargetProcesses, MpiWrapper::commSize(), + GEOS_FMT( "{}: processes count cannot exceed the launched ranks count.", + getWrapperDataContext( viewKeysStruct::numberOfTargetProcesses ) ) ); + m_writer.setNumberOfTargetProcesses( m_numberOfTargetProcesses ); + string const fieldNamesString = viewKeysStruct::fieldNames; string const onlyPlotSpecifiedFieldNamesString = viewKeysStruct::onlyPlotSpecifiedFieldNames; diff --git a/src/coreComponents/fileIO/Outputs/VTKOutput.hpp b/src/coreComponents/fileIO/Outputs/VTKOutput.hpp index 871a6497569..5b167dc66d6 100644 --- a/src/coreComponents/fileIO/Outputs/VTKOutput.hpp +++ b/src/coreComponents/fileIO/Outputs/VTKOutput.hpp @@ -97,6 +97,7 @@ class VTKOutput : public OutputBase static constexpr auto onlyPlotSpecifiedFieldNames = "onlyPlotSpecifiedFieldNames"; static constexpr auto fieldNames = "fieldNames"; static constexpr auto levelNames = "levelNames"; + static constexpr auto numberOfTargetProcesses = "numberOfTargetProcesses"; } vtkOutputViewKeys; /// @endcond @@ -114,6 +115,9 @@ class VTKOutput : public OutputBase integer m_writeFaceMesh; integer m_plotLevel; + /// Aggregate output data to be written + integer m_numberOfTargetProcesses; + /// Should the vtk files contain the ghost cells or not. integer m_writeGhostCells; diff --git a/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.cpp b/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.cpp index be65508921b..5f063a4e3ca 100644 --- a/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.cpp +++ b/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.cpp @@ -33,11 +33,14 @@ #include #include #include - +#include // System includes #include #include +#include "mesh/generators/VTKUtilities.hpp" + + namespace geos { @@ -1041,7 +1044,7 @@ void VTKPolyDataWriterInterface::writeElementFields( ElementRegionBase const & r void VTKPolyDataWriterInterface::writeCellElementRegions( real64 const time, ElementRegionManager const & elemManager, NodeManager const & nodeManager, - string const & path ) const + string const & path ) { elemManager.forElementRegions< CellElementRegion >( [&]( CellElementRegion const & region ) { @@ -1055,15 +1058,13 @@ void VTKPolyDataWriterInterface::writeCellElementRegions( real64 const time, writeTimestamp( ug.GetPointer(), time ); writeElementFields( region, ug->GetCellData() ); writeNodeFields( nodeManager, VTKCells.nodes, ug->GetPointData() ); - - string const regionDir = joinPath( path, region.getName() ); - writeUnstructuredGrid( regionDir, ug.GetPointer() ); + writeUnstructuredGrid( path, region, ug.GetPointer() ); } ); } void VTKPolyDataWriterInterface::writeParticleRegions( real64 const time, ParticleManager const & particleManager, - string const & path ) const + string const & path ) { particleManager.forParticleRegions< ParticleRegion >( [&]( ParticleRegion const & region ) { @@ -1077,15 +1078,15 @@ void VTKPolyDataWriterInterface::writeParticleRegions( real64 const time, writeTimestamp( ug.GetPointer(), time ); writeParticleFields( region, ug->GetCellData() ); - string const regionDir = joinPath( path, region.getName() ); - writeUnstructuredGrid( regionDir, ug.GetPointer() ); + // string const regionDir = joinPath( path, region.getName() ); + writeUnstructuredGrid( path, region, ug.GetPointer() ); } ); } void VTKPolyDataWriterInterface::writeWellElementRegions( real64 const time, ElementRegionManager const & elemManager, NodeManager const & nodeManager, - string const & path ) const + string const & path ) { elemManager.forElementRegions< WellElementRegion >( [&]( WellElementRegion const & region ) { @@ -1098,9 +1099,7 @@ void VTKPolyDataWriterInterface::writeWellElementRegions( real64 const time, writeTimestamp( ug.GetPointer(), time ); writeElementFields( region, ug->GetCellData() ); - - string const regionDir = joinPath( path, region.getName() ); - writeUnstructuredGrid( regionDir, ug.GetPointer() ); + writeUnstructuredGrid( path, region, ug.GetPointer() ); } ); } @@ -1109,7 +1108,7 @@ void VTKPolyDataWriterInterface::writeSurfaceElementRegions( real64 const time, NodeManager const & nodeManager, EmbeddedSurfaceNodeManager const & embSurfNodeManager, FaceManager const & faceManager, - string const & path ) const + string const & path ) { elemManager.forElementRegions< SurfaceElementRegion >( [&]( SurfaceElementRegion const & region ) { @@ -1140,9 +1139,7 @@ void VTKPolyDataWriterInterface::writeSurfaceElementRegions( real64 const time, writeTimestamp( ug.GetPointer(), time ); writeElementFields( region, ug->GetCellData() ); - - string const regionDir = joinPath( path, region.getName() ); - writeUnstructuredGrid( regionDir, ug.GetPointer() ); + writeUnstructuredGrid( path, region, ug.GetPointer() ); } ); } @@ -1188,13 +1185,13 @@ void VTKPolyDataWriterInterface::writeVtmFile( integer const cycle, string const meshPath = joinPath( getCycleSubFolder( cycle ), meshBodyName, meshLevelName ); - int const mpiSize = MpiWrapper::commSize(); + // int const mpiSize = MpiWrapper::commSize(); auto addElementRegion = [&]( ElementRegionBase const & region ) { std::vector< string > const blockPath{ meshBody.getName(), meshLevel.getName(), region.getCatalogName(), region.getName() }; string const regionPath = joinPath( meshPath, region.getName() ); - for( int i = 0; i < mpiSize; i++ ) + for( const auto & i : m_targetProcessesId.at( region.getName()) ) { string const dataSetName = getRankFileName( i ); string const dataSetFile = joinPath( regionPath, dataSetName + ".vtu" ); @@ -1207,7 +1204,7 @@ void VTKPolyDataWriterInterface::writeVtmFile( integer const cycle, string const & regionName = region.getName(); std::vector< string > const blockPath{ meshBodyName, meshLevelName, region.getCatalogName(), regionName }; string const regionPath = joinPath( meshPath, regionName ); - for( int i = 0; i < mpiSize; i++ ) + for( const auto & i : m_targetProcessesId.at( region.getName()) ) { string const dataSetName = getRankFileName( i ); string const dataSetFile = joinPath( regionPath, dataSetName + ".vtu" ); @@ -1256,8 +1253,11 @@ int toVtkOutputMode( VTKOutputMode const mode ) } void VTKPolyDataWriterInterface::writeUnstructuredGrid( string const & path, - vtkUnstructuredGrid * ug ) const + ObjectManagerBase const & region, + vtkUnstructuredGrid * ug ) { + string const regionDir = joinPath( path, region.getName() ); + vtkSmartPointer< vtkAlgorithm > filter; // If we want to get rid of the ghost ranks, we use the appropriate `vtkThreshold` filter. @@ -1279,15 +1279,51 @@ void VTKPolyDataWriterInterface::writeUnstructuredGrid( string const & path, } filter->SetInputDataObject( ug ); - filter->Update(); - - makeDirectory( path ); - string const vtuFilePath = joinPath( path, getRankFileName( MpiWrapper::commRank() ) + ".vtu" ); - auto const vtuWriter = vtkSmartPointer< vtkXMLUnstructuredGridWriter >::New(); - vtuWriter->SetInputData( filter->GetOutputDataObject( 0 ) ); - vtuWriter->SetFileName( vtuFilePath.c_str() ); - vtuWriter->SetDataMode( toVtkOutputMode( m_outputMode ) ); - vtuWriter->Write(); + + vtkSmartPointer< vtkMultiProcessController > controller = vtk::getController(); + vtkMultiProcessController::SetGlobalController( controller ); + + // In case of m_numberOfTargetProcesses == GetNumberOfProcesses the filter returns a shallow copy + // The behavior is the same as previously in this case. The rank number is computed instead of implicitly written + vtkNew< vtkAggregateDataSetFilter > aggregate; + aggregate->SetInputConnection( filter->GetOutputPort()); + aggregate->SetNumberOfTargetProcesses( m_numberOfTargetProcesses ); + aggregate->SetMergePoints( false ); + aggregate->Update(); + + int localCommRank = -1; + if( vtkDataSet::SafeDownCast( aggregate->GetOutput())->GetNumberOfPoints() != 0 ) + { + localCommRank = MpiWrapper::commRank(); + makeDirectory( regionDir ); + string const vtuFilePath = joinPath( regionDir, getRankFileName( localCommRank ) + ".vtu" ); + auto const vtuWriter = vtkSmartPointer< vtkXMLUnstructuredGridWriter >::New(); + vtuWriter->SetInputData( aggregate->GetOutput() ); + vtuWriter->SetFileName( vtuFilePath.c_str() ); + vtuWriter->SetDataMode( toVtkOutputMode( m_outputMode ) ); + vtuWriter->Write(); + } + + const int size = MpiWrapper::commSize( MPI_COMM_GEOS ); + std::vector< int > globalValues( size ); + + // Everything is done on rank 0 + MpiWrapper::gather( &localCommRank, + 1, + globalValues.data(), + 1, + 0, + MPI_COMM_GEOS ); + + if( MpiWrapper::commRank() == 0 ) + { + // any rank that does not hold data will not participate in the output + globalValues.erase( std::remove_if( globalValues.begin(), + globalValues.end(), + []( int x ) { return x == -1; } ), + globalValues.end()); + m_targetProcessesId[region.getName()] = globalValues; + } } void VTKPolyDataWriterInterface::write( real64 const time, diff --git a/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp b/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp index 77c6b90e8aa..44bb86e3c71 100644 --- a/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp +++ b/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp @@ -17,6 +17,7 @@ #define GEOS_FILEIO_VTK_VTKPOLYDATAWRITERINTERFACE_HPP_ #include "common/DataTypes.hpp" +#include "mesh/ObjectManagerBase.hpp" #include "dataRepository/WrapperBase.hpp" #include "dataRepository/Wrapper.hpp" #include "fileIO/vtk/VTKPVDWriter.hpp" @@ -167,6 +168,14 @@ class VTKPolyDataWriterInterface { m_levelNames.insert( levelNames.begin(), levelNames.end() ); } + /** + * @brief Set the Number Of Target Processes + * @param[in] numberOfTargetProcesses the number of processes + */ + void setNumberOfTargetProcesses( integer const numberOfTargetProcesses ) + { + m_numberOfTargetProcesses = numberOfTargetProcesses; + } /** * @brief Main method of this class. Write all the files for one time step. @@ -223,33 +232,38 @@ class VTKPolyDataWriterInterface * @brief Writes the files for all the CellElementRegions. * @details There will be one file written per CellElementRegion and per rank. * @param[in] time the time-step - * @param[in] cycle the current cycle number * @param[in] elemManager the ElementRegionManager containing the CellElementRegions to be output * @param[in] nodeManager the NodeManager containing the nodes of the domain to be output - * @param[in] meshLevelName the name of the MeshLevel containing the nodes and elements to be output - * @param[in] meshBodyName the name of the MeshBody containing the nodes and elements to be output + * @param[in] path the root path where the mesh will be written */ void writeCellElementRegions( real64 time, ElementRegionManager const & elemManager, NodeManager const & nodeManager, - string const & path ) const; - - void writeParticleRegions( real64 const time, - ParticleManager const & particleManager, - string const & path ) const; + string const & path ); /** * @brief Writes the files containing the well representation * @details There will be one file written per WellElementRegion and per rank * @param[in] time the time-step - * @param[in] cycle the current cycle number * @param[in] elemManager the ElementRegionManager containing the WellElementRegions to be output * @param[in] nodeManager the NodeManager containing the nodes of the domain to be output + * @param[in] path the root path where the mesh will be written */ void writeWellElementRegions( real64 time, ElementRegionManager const & elemManager, NodeManager const & nodeManager, - string const & path ) const; + string const & path ); + + /** + * @brief Writes the files containing the particle representation + * @details There will be one file written per ParticleRegion and per rank + * @param[in] time the time-step + * @param[in] particleManager the ParticleManager containing the ParticleRegions to be output + * @param[in] path the root path where the mesh will be written + */ + void writeParticleRegions( real64 const time, + ParticleManager const & particleManager, + string const & path ); /** * @brief Writes the files containing the faces elements @@ -266,13 +280,13 @@ class VTKPolyDataWriterInterface NodeManager const & nodeManager, EmbeddedSurfaceNodeManager const & embSurfNodeManager, FaceManager const & faceManager, - string const & path ) const; + string const & path ); /** * @brief Writes a VTM file for the time-step \p time. * @details a VTM file is a VTK Multiblock file. It contains relative path to different files organized in blocks. * @param[in] cycle the current cycle number - * @param[in] elemManager the ElementRegionManager containing all the regions to be output and referred to in the VTM file + * @param[in] domain the DomainPartition containing all the regions to be output and referred to in the VTM file * @param[in] vtmWriter a writer specialized for the VTM file format */ @@ -297,6 +311,11 @@ class VTKPolyDataWriterInterface void writeElementFields( ElementRegionBase const & subRegion, vtkCellData * cellData ) const; + /** + * @brief Writes all the fields associated to the elements of \p er if their plotlevel is <= m_plotLevel + * @param[in] region ParticleRegion being written + * @param[in] cellData a VTK object containing all the fields associated with the elements + */ void writeParticleFields( ParticleRegionBase const & region, vtkCellData * cellData ) const; @@ -305,11 +324,13 @@ class VTKPolyDataWriterInterface * @details The unstructured grid is the last element in the hierarchy of the output, * it contains the cells connectivities and the vertices coordinates as long as the * data fields associated with it - * @param[in] ug a VTK SmartPointer to the VTK unstructured grid. * @param[in] path directory path for the grid file + * @param[in] region ElementRegionBase beeing written + * @param[in] ug a VTK SmartPointer to the VTK unstructured grid. */ void writeUnstructuredGrid( string const & path, - vtkUnstructuredGrid * ug ) const; + ObjectManagerBase const & region, + vtkUnstructuredGrid * ug ); private: @@ -352,6 +373,12 @@ class VTKPolyDataWriterInterface /// Defines whether to plot a faceElement as a 3D volumetric element or not. bool m_writeFaceElementsAs3D; + + /// Number of target processes to aggregate the data to be written + integer m_numberOfTargetProcesses; + + /// Map a region name to the array of ranks outputed for it + std::map< string, std::vector< integer > > m_targetProcessesId; }; } // namespace vtk diff --git a/src/coreComponents/schema/schema.xsd b/src/coreComponents/schema/schema.xsd index 3235c8a9794..e013bde4576 100644 --- a/src/coreComponents/schema/schema.xsd +++ b/src/coreComponents/schema/schema.xsd @@ -2149,6 +2149,8 @@ the relative residual norm satisfies: + +