feat: choose the number of output vtu files (#2637)

Co-authored-by: MelReyCG <[email protected]>

.integrated_tests.yaml

@@ -1,6 +1,6 @@
 baselines:
   bucket: geosx
-  baseline: integratedTests/baseline_integratedTests-pr3439-8919-b09db43
+  baseline: integratedTests/baseline_integratedTests-pr2637-8940-3c0fc5c
 allow_fail:
   all: ''
   streak: ''

BASELINE_NOTES.md

@@ -6,6 +6,10 @@ This file is designed to track changes to the integrated test baselines.
 Any developer who updates the baseline ID in the .integrated_tests.yaml file is expected to create an entry in this file with the pull request number, date, and their justification for rebaselining.
 These notes should be in reverse-chronological order, and use the following time format: (YYYY-MM-DD).
 
+PR #2637 (2024-11-21)
+=====================
+Added numberOfTargetProcesses.
+
 PR #3439 (2024-11-20)
 =====================
 EDFM bugfixes: derivatives sign, frac/cell element volume, fix apertures inconsistency in test cases.

src/coreComponents/fileIO/Outputs/VTKOutput.cpp

@@ -18,7 +18,7 @@
  */
 
 #include "VTKOutput.hpp"
-
+#include "common/MpiWrapper.hpp"
 
 #if defined(GEOS_USE_PYGEOSX)
 #include "fileIO/python/PyVTKOutputType.hpp"
@@ -56,6 +56,11 @@ VTKOutput::VTKOutput( string const & name,
     setInputFlag( InputFlags::OPTIONAL ).
     setDescription( "Level detail plot. Only fields with lower of equal plot level will be output." );
 
+  registerWrapper( viewKeysStruct::numberOfTargetProcesses, &m_numberOfTargetProcesses ).
+    setApplyDefaultValue( MpiWrapper::commSize() ).
+    setInputFlag( InputFlags::OPTIONAL ).
+    setDescription( "Number of output aggregate files to be written." );
+
   registerWrapper( viewKeysStruct::writeGhostCells, &m_writeGhostCells ).
     setApplyDefaultValue( 0 ).
     setInputFlag( InputFlags::OPTIONAL ).
@@ -102,6 +107,14 @@ void VTKOutput::postInputInitialization()
   m_writer.setLevelNames( m_levelNames.toViewConst() );
   m_writer.setOnlyPlotSpecifiedFieldNamesFlag( m_onlyPlotSpecifiedFieldNames );
 
+  GEOS_ERROR_IF_LT_MSG( m_numberOfTargetProcesses, 1,
+                        GEOS_FMT( "{}: processes count cannot be less than 1.",
+                                  getWrapperDataContext( viewKeysStruct::numberOfTargetProcesses ) ) );
+  GEOS_ERROR_IF_GT_MSG( m_numberOfTargetProcesses, MpiWrapper::commSize(),
+                        GEOS_FMT( "{}: processes count cannot exceed the launched ranks count.",
+                                  getWrapperDataContext( viewKeysStruct::numberOfTargetProcesses ) ) );
+  m_writer.setNumberOfTargetProcesses( m_numberOfTargetProcesses );
+
   string const fieldNamesString = viewKeysStruct::fieldNames;
   string const onlyPlotSpecifiedFieldNamesString = viewKeysStruct::onlyPlotSpecifiedFieldNames;
 

src/coreComponents/fileIO/Outputs/VTKOutput.hpp

@@ -97,6 +97,7 @@ class VTKOutput : public OutputBase
     static constexpr auto onlyPlotSpecifiedFieldNames = "onlyPlotSpecifiedFieldNames";
     static constexpr auto fieldNames = "fieldNames";
     static constexpr auto levelNames = "levelNames";
+    static constexpr auto numberOfTargetProcesses = "numberOfTargetProcesses";
   } vtkOutputViewKeys;
   /// @endcond
 
@@ -114,6 +115,9 @@ class VTKOutput : public OutputBase
   integer m_writeFaceMesh;
   integer m_plotLevel;
 
+  /// Aggregate output data to be written
+  integer m_numberOfTargetProcesses;
+
   /// Should the vtk files contain the ghost cells or not.
   integer m_writeGhostCells;
 

src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.cpp

@@ -33,11 +33,14 @@
 #include <vtkThreshold.h>
 #include <vtkUnstructuredGrid.h>
 #include <vtkXMLUnstructuredGridWriter.h>
-
+#include <vtkAggregateDataSetFilter.h>
 // System includes
 #include <numeric>
 #include <unordered_set>
 
+#include "mesh/generators/VTKUtilities.hpp"
+
+
 namespace geos
 {
 
@@ -1041,7 +1044,7 @@ void VTKPolyDataWriterInterface::writeElementFields( ElementRegionBase const & r
 void VTKPolyDataWriterInterface::writeCellElementRegions( real64 const time,
                                                           ElementRegionManager const & elemManager,
                                                           NodeManager const & nodeManager,
-                                                          string const & path ) const
+                                                          string const & path )
 {
   elemManager.forElementRegions< CellElementRegion >( [&]( CellElementRegion const & region )
   {
@@ -1055,15 +1058,13 @@ void VTKPolyDataWriterInterface::writeCellElementRegions( real64 const time,
     writeTimestamp( ug.GetPointer(), time );
     writeElementFields( region, ug->GetCellData() );
     writeNodeFields( nodeManager, VTKCells.nodes, ug->GetPointData() );
-
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
 void VTKPolyDataWriterInterface::writeParticleRegions( real64 const time,
                                                        ParticleManager const & particleManager,
-                                                       string const & path ) const
+                                                       string const & path )
 {
   particleManager.forParticleRegions< ParticleRegion >( [&]( ParticleRegion const & region )
   {
@@ -1077,15 +1078,15 @@ void VTKPolyDataWriterInterface::writeParticleRegions( real64 const time,
     writeTimestamp( ug.GetPointer(), time );
     writeParticleFields( region, ug->GetCellData() );
 
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    // string const regionDir = joinPath( path, region.getName() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
 void VTKPolyDataWriterInterface::writeWellElementRegions( real64 const time,
                                                           ElementRegionManager const & elemManager,
                                                           NodeManager const & nodeManager,
-                                                          string const & path ) const
+                                                          string const & path )
 {
   elemManager.forElementRegions< WellElementRegion >( [&]( WellElementRegion const & region )
   {
@@ -1098,9 +1099,7 @@ void VTKPolyDataWriterInterface::writeWellElementRegions( real64 const time,
 
     writeTimestamp( ug.GetPointer(), time );
     writeElementFields( region, ug->GetCellData() );
-
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
@@ -1109,7 +1108,7 @@ void VTKPolyDataWriterInterface::writeSurfaceElementRegions( real64 const time,
                                                              NodeManager const & nodeManager,
                                                              EmbeddedSurfaceNodeManager const & embSurfNodeManager,
                                                              FaceManager const & faceManager,
-                                                             string const & path ) const
+                                                             string const & path )
 {
   elemManager.forElementRegions< SurfaceElementRegion >( [&]( SurfaceElementRegion const & region )
   {
@@ -1140,9 +1139,7 @@ void VTKPolyDataWriterInterface::writeSurfaceElementRegions( real64 const time,
 
     writeTimestamp( ug.GetPointer(), time );
     writeElementFields( region, ug->GetCellData() );
-
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
@@ -1188,13 +1185,13 @@ void VTKPolyDataWriterInterface::writeVtmFile( integer const cycle,
 
       string const meshPath = joinPath( getCycleSubFolder( cycle ), meshBodyName, meshLevelName );
 
-      int const mpiSize = MpiWrapper::commSize();
+      // int const mpiSize = MpiWrapper::commSize();
 
       auto addElementRegion = [&]( ElementRegionBase const & region )
       {
         std::vector< string > const blockPath{ meshBody.getName(), meshLevel.getName(), region.getCatalogName(), region.getName() };
         string const regionPath = joinPath( meshPath, region.getName() );
-        for( int i = 0; i < mpiSize; i++ )
+        for( const auto & i : m_targetProcessesId.at( region.getName()) )
         {
           string const dataSetName = getRankFileName( i );
           string const dataSetFile = joinPath( regionPath, dataSetName + ".vtu" );
@@ -1207,7 +1204,7 @@ void VTKPolyDataWriterInterface::writeVtmFile( integer const cycle,
         string const & regionName = region.getName();
         std::vector< string > const blockPath{ meshBodyName, meshLevelName, region.getCatalogName(), regionName };
         string const regionPath = joinPath( meshPath, regionName );
-        for( int i = 0; i < mpiSize; i++ )
+        for( const auto & i : m_targetProcessesId.at( region.getName()) )
         {
           string const dataSetName = getRankFileName( i );
           string const dataSetFile = joinPath( regionPath, dataSetName + ".vtu" );
@@ -1256,8 +1253,11 @@ int toVtkOutputMode( VTKOutputMode const mode )
 }
 
 void VTKPolyDataWriterInterface::writeUnstructuredGrid( string const & path,
-                                                        vtkUnstructuredGrid * ug ) const
+                                                        ObjectManagerBase const & region,
+                                                        vtkUnstructuredGrid * ug )
 {
+  string const regionDir = joinPath( path, region.getName() );
+
   vtkSmartPointer< vtkAlgorithm > filter;
 
   // If we want to get rid of the ghost ranks, we use the appropriate `vtkThreshold` filter.
@@ -1279,15 +1279,51 @@ void VTKPolyDataWriterInterface::writeUnstructuredGrid( string const & path,
   }
 
   filter->SetInputDataObject( ug );
-  filter->Update();
-
-  makeDirectory( path );
-  string const vtuFilePath = joinPath( path, getRankFileName( MpiWrapper::commRank() ) + ".vtu" );
-  auto const vtuWriter = vtkSmartPointer< vtkXMLUnstructuredGridWriter >::New();
-  vtuWriter->SetInputData( filter->GetOutputDataObject( 0 ) );
-  vtuWriter->SetFileName( vtuFilePath.c_str() );
-  vtuWriter->SetDataMode( toVtkOutputMode( m_outputMode ) );
-  vtuWriter->Write();
+
+  vtkSmartPointer< vtkMultiProcessController > controller = vtk::getController();
+  vtkMultiProcessController::SetGlobalController( controller );
+
+  // In case of m_numberOfTargetProcesses == GetNumberOfProcesses the filter returns a shallow copy
+  // The behavior  is the same as previously in this case. The rank number is computed instead of implicitly written
+  vtkNew< vtkAggregateDataSetFilter > aggregate;
+  aggregate->SetInputConnection( filter->GetOutputPort());
+  aggregate->SetNumberOfTargetProcesses( m_numberOfTargetProcesses );
+  aggregate->SetMergePoints( false );
+  aggregate->Update();
+
+  int localCommRank = -1;
+  if( vtkDataSet::SafeDownCast( aggregate->GetOutput())->GetNumberOfPoints() != 0 )
+  {
+    localCommRank = MpiWrapper::commRank();
+    makeDirectory( regionDir );
+    string const vtuFilePath = joinPath( regionDir, getRankFileName( localCommRank ) + ".vtu" );
+    auto const vtuWriter = vtkSmartPointer< vtkXMLUnstructuredGridWriter >::New();
+    vtuWriter->SetInputData( aggregate->GetOutput() );
+    vtuWriter->SetFileName( vtuFilePath.c_str() );
+    vtuWriter->SetDataMode( toVtkOutputMode( m_outputMode ) );
+    vtuWriter->Write();
+  }
+
+  const int size = MpiWrapper::commSize( MPI_COMM_GEOS );
+  std::vector< int > globalValues( size );
+
+  // Everything is done on rank 0
+  MpiWrapper::gather( &localCommRank,
+                      1,
+                      globalValues.data(),
+                      1,
+                      0,
+                      MPI_COMM_GEOS );
+
+  if( MpiWrapper::commRank() == 0 )
+  {
+    // any rank that does not hold data will not participate in the output
+    globalValues.erase( std::remove_if( globalValues.begin(),
+                                        globalValues.end(),
+                                        []( int x ) { return x == -1; } ),
+                        globalValues.end());
+    m_targetProcessesId[region.getName()] = globalValues;
+  }
 }
 
 void VTKPolyDataWriterInterface::write( real64 const time,

src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp

@@ -17,6 +17,7 @@
 #define GEOS_FILEIO_VTK_VTKPOLYDATAWRITERINTERFACE_HPP_
 
 #include "common/DataTypes.hpp"
+#include "mesh/ObjectManagerBase.hpp"
 #include "dataRepository/WrapperBase.hpp"
 #include "dataRepository/Wrapper.hpp"
 #include "fileIO/vtk/VTKPVDWriter.hpp"
@@ -167,6 +168,14 @@ class VTKPolyDataWriterInterface
   {
     m_levelNames.insert( levelNames.begin(), levelNames.end() );
   }
+  /**
+   * @brief Set the Number Of Target Processes
+   * @param[in] numberOfTargetProcesses  the number of processes
+   */
+  void setNumberOfTargetProcesses( integer const numberOfTargetProcesses )
+  {
+    m_numberOfTargetProcesses = numberOfTargetProcesses;
+  }
 
   /**
    * @brief Main method of this class. Write all the files for one time step.
@@ -223,33 +232,38 @@ class VTKPolyDataWriterInterface
    * @brief Writes the files for all the CellElementRegions.
    * @details There will be one file written per CellElementRegion and per rank.
    * @param[in] time the time-step
-   * @param[in] cycle the current cycle number
    * @param[in] elemManager the ElementRegionManager containing the CellElementRegions to be output
    * @param[in] nodeManager the NodeManager containing the nodes of the domain to be output
-   * @param[in] meshLevelName the name of the MeshLevel containing the nodes and elements to be output
-   * @param[in] meshBodyName the name of the MeshBody containing the nodes and elements to be output
+   * @param[in] path the root path where the mesh will be written
    */
   void writeCellElementRegions( real64 time,
                                 ElementRegionManager const & elemManager,
                                 NodeManager const & nodeManager,
-                                string const & path ) const;
-
-  void writeParticleRegions( real64 const time,
-                             ParticleManager const & particleManager,
-                             string const & path ) const;
+                                string const & path );
 
   /**
    * @brief Writes the files containing the well representation
    * @details There will be one file written per WellElementRegion and per rank
    * @param[in] time the time-step
-   * @param[in] cycle the current cycle number
    * @param[in] elemManager the ElementRegionManager containing the WellElementRegions to be output
    * @param[in] nodeManager the NodeManager containing the nodes of the domain to be output
+   * @param[in] path the root path where the mesh will be written
    */
   void writeWellElementRegions( real64 time,
                                 ElementRegionManager const & elemManager,
                                 NodeManager const & nodeManager,
-                                string const & path ) const;
+                                string const & path );
+
+  /**
+   * @brief Writes the files containing the particle representation
+   * @details There will be one file written per ParticleRegion and per rank
+   * @param[in] time the time-step
+   * @param[in] particleManager the ParticleManager containing the ParticleRegions to be output
+   * @param[in] path the root path where the mesh will be written
+   */
+  void writeParticleRegions( real64 const time,
+                             ParticleManager const & particleManager,
+                             string const & path );
 
   /**
    * @brief Writes the files containing the faces elements
@@ -266,13 +280,13 @@ class VTKPolyDataWriterInterface
                                    NodeManager const & nodeManager,
                                    EmbeddedSurfaceNodeManager const & embSurfNodeManager,
                                    FaceManager const & faceManager,
-                                   string const & path ) const;
+                                   string const & path );
 
   /**
    * @brief Writes a VTM file for the time-step \p time.
    * @details a VTM file is a VTK Multiblock file. It contains relative path to different files organized in blocks.
    * @param[in] cycle the current cycle number
-   * @param[in] elemManager the ElementRegionManager containing all the regions to be output and referred to in the VTM file
+   * @param[in] domain the DomainPartition containing all the regions to be output and referred to in the VTM file
    * @param[in] vtmWriter a writer specialized for the VTM file format
    */
 
@@ -297,6 +311,11 @@ class VTKPolyDataWriterInterface
   void writeElementFields( ElementRegionBase const & subRegion,
                            vtkCellData * cellData ) const;
 
+  /**
+   * @brief Writes all the fields associated to the elements of \p er if their plotlevel is <= m_plotLevel
+   * @param[in] region ParticleRegion being written
+   * @param[in] cellData a VTK object containing all the fields associated with the elements
+   */
   void writeParticleFields( ParticleRegionBase const & region,
                             vtkCellData * cellData ) const;
 
@@ -305,11 +324,13 @@ class VTKPolyDataWriterInterface
    * @details The unstructured grid is the last element in the hierarchy of the output,
    * it contains the cells connectivities and the vertices coordinates as long as the
    * data fields associated with it
-   * @param[in] ug a VTK SmartPointer to the VTK unstructured grid.
    * @param[in] path directory path for the grid file
+   * @param[in] region ElementRegionBase beeing written
+   * @param[in] ug a VTK SmartPointer to the VTK unstructured grid.
    */
   void writeUnstructuredGrid( string const & path,
-                              vtkUnstructuredGrid * ug ) const;
+                              ObjectManagerBase const & region,
+                              vtkUnstructuredGrid * ug );
 
 private:
 
@@ -352,6 +373,12 @@ class VTKPolyDataWriterInterface
 
   /// Defines whether to plot a faceElement as a 3D volumetric element or not.
   bool m_writeFaceElementsAs3D;
+
+  /// Number of target processes to aggregate the data to be written
+  integer m_numberOfTargetProcesses;
+
+  /// Map a region name to the array of ranks outputed for it
+  std::map< string, std::vector< integer > > m_targetProcessesId;
 };
 
 } // namespace vtk