Merge branch 'develop' into pt/deltaVolume-fix

.integrated_tests.yaml

@@ -1,6 +1,6 @@
 baselines:
   bucket: geosx
-  baseline: integratedTests/baseline_integratedTests-pr3439-8919-b09db43
+  baseline: integratedTests/baseline_integratedTests-pr2637-8940-3c0fc5c
 allow_fail:
   all: ''
   streak: ''

BASELINE_NOTES.md

@@ -6,6 +6,10 @@ This file is designed to track changes to the integrated test baselines.
 Any developer who updates the baseline ID in the .integrated_tests.yaml file is expected to create an entry in this file with the pull request number, date, and their justification for rebaselining.
 These notes should be in reverse-chronological order, and use the following time format: (YYYY-MM-DD).
 
+PR #2637 (2024-11-21)
+=====================
+Added numberOfTargetProcesses.
+
 PR #3439 (2024-11-20)
 =====================
 EDFM bugfixes: derivatives sign, frac/cell element volume, fix apertures inconsistency in test cases.

src/cmake/thirdparty/SetupGeosxThirdParty.cmake

@@ -670,7 +670,7 @@ if(DEFINED HYPRE_DIR AND ENABLE_HYPRE)
         find_package( rocsolver REQUIRED )
         find_package( rocsparse REQUIRED )
         find_package( rocrand REQUIRED )
-        append( APPEND HYPRE_DEPENDS roc::rocblas roc::rocsparse roc::rocsolver roc::rocrand )
+        list( APPEND HYPRE_DEPENDS roc::rocblas roc::rocsparse roc::rocsolver roc::rocrand )
     endif( )
 
     find_and_import( NAME hypre

src/coreComponents/common/format/StringUtilities.cpp

@@ -93,8 +93,9 @@ string addCommaSeparators( T const & num )
   return result;
 }
 
-template string addCommaSeparators( localIndex const & num );
-template string addCommaSeparators( globalIndex const & num );
+template string addCommaSeparators( int const & num );
+template string addCommaSeparators( long int const & num );
+template string addCommaSeparators( long long int const & num );
 
 // put definition here so we can control the allowable values of T and
 // modication of this function triggers a whole code recompile...which

src/coreComponents/fileIO/Outputs/VTKOutput.cpp

@@ -18,7 +18,7 @@
  */
 
 #include "VTKOutput.hpp"
-
+#include "common/MpiWrapper.hpp"
 
 #if defined(GEOS_USE_PYGEOSX)
 #include "fileIO/python/PyVTKOutputType.hpp"
@@ -56,6 +56,11 @@ VTKOutput::VTKOutput( string const & name,
     setInputFlag( InputFlags::OPTIONAL ).
     setDescription( "Level detail plot. Only fields with lower of equal plot level will be output." );
 
+  registerWrapper( viewKeysStruct::numberOfTargetProcesses, &m_numberOfTargetProcesses ).
+    setApplyDefaultValue( MpiWrapper::commSize() ).
+    setInputFlag( InputFlags::OPTIONAL ).
+    setDescription( "Number of output aggregate files to be written." );
+
   registerWrapper( viewKeysStruct::writeGhostCells, &m_writeGhostCells ).
     setApplyDefaultValue( 0 ).
     setInputFlag( InputFlags::OPTIONAL ).
@@ -102,6 +107,14 @@ void VTKOutput::postInputInitialization()
   m_writer.setLevelNames( m_levelNames.toViewConst() );
   m_writer.setOnlyPlotSpecifiedFieldNamesFlag( m_onlyPlotSpecifiedFieldNames );
 
+  GEOS_ERROR_IF_LT_MSG( m_numberOfTargetProcesses, 1,
+                        GEOS_FMT( "{}: processes count cannot be less than 1.",
+                                  getWrapperDataContext( viewKeysStruct::numberOfTargetProcesses ) ) );
+  GEOS_ERROR_IF_GT_MSG( m_numberOfTargetProcesses, MpiWrapper::commSize(),
+                        GEOS_FMT( "{}: processes count cannot exceed the launched ranks count.",
+                                  getWrapperDataContext( viewKeysStruct::numberOfTargetProcesses ) ) );
+  m_writer.setNumberOfTargetProcesses( m_numberOfTargetProcesses );
+
   string const fieldNamesString = viewKeysStruct::fieldNames;
   string const onlyPlotSpecifiedFieldNamesString = viewKeysStruct::onlyPlotSpecifiedFieldNames;
 

src/coreComponents/fileIO/Outputs/VTKOutput.hpp

@@ -97,6 +97,7 @@ class VTKOutput : public OutputBase
     static constexpr auto onlyPlotSpecifiedFieldNames = "onlyPlotSpecifiedFieldNames";
     static constexpr auto fieldNames = "fieldNames";
     static constexpr auto levelNames = "levelNames";
+    static constexpr auto numberOfTargetProcesses = "numberOfTargetProcesses";
   } vtkOutputViewKeys;
   /// @endcond
 
@@ -114,6 +115,9 @@ class VTKOutput : public OutputBase
   integer m_writeFaceMesh;
   integer m_plotLevel;
 
+  /// Aggregate output data to be written
+  integer m_numberOfTargetProcesses;
+
   /// Should the vtk files contain the ghost cells or not.
   integer m_writeGhostCells;
 

src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.cpp

@@ -33,11 +33,14 @@
 #include <vtkThreshold.h>
 #include <vtkUnstructuredGrid.h>
 #include <vtkXMLUnstructuredGridWriter.h>
-
+#include <vtkAggregateDataSetFilter.h>
 // System includes
 #include <numeric>
 #include <unordered_set>
 
+#include "mesh/generators/VTKUtilities.hpp"
+
+
 namespace geos
 {
 
@@ -1041,7 +1044,7 @@ void VTKPolyDataWriterInterface::writeElementFields( ElementRegionBase const & r
 void VTKPolyDataWriterInterface::writeCellElementRegions( real64 const time,
                                                           ElementRegionManager const & elemManager,
                                                           NodeManager const & nodeManager,
-                                                          string const & path ) const
+                                                          string const & path )
 {
   elemManager.forElementRegions< CellElementRegion >( [&]( CellElementRegion const & region )
   {
@@ -1055,15 +1058,13 @@ void VTKPolyDataWriterInterface::writeCellElementRegions( real64 const time,
     writeTimestamp( ug.GetPointer(), time );
     writeElementFields( region, ug->GetCellData() );
     writeNodeFields( nodeManager, VTKCells.nodes, ug->GetPointData() );
-
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
 void VTKPolyDataWriterInterface::writeParticleRegions( real64 const time,
                                                        ParticleManager const & particleManager,
-                                                       string const & path ) const
+                                                       string const & path )
 {
   particleManager.forParticleRegions< ParticleRegion >( [&]( ParticleRegion const & region )
   {
@@ -1077,15 +1078,15 @@ void VTKPolyDataWriterInterface::writeParticleRegions( real64 const time,
     writeTimestamp( ug.GetPointer(), time );
     writeParticleFields( region, ug->GetCellData() );
 
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    // string const regionDir = joinPath( path, region.getName() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
 void VTKPolyDataWriterInterface::writeWellElementRegions( real64 const time,
                                                           ElementRegionManager const & elemManager,
                                                           NodeManager const & nodeManager,
-                                                          string const & path ) const
+                                                          string const & path )
 {
   elemManager.forElementRegions< WellElementRegion >( [&]( WellElementRegion const & region )
   {
@@ -1098,9 +1099,7 @@ void VTKPolyDataWriterInterface::writeWellElementRegions( real64 const time,
 
     writeTimestamp( ug.GetPointer(), time );
     writeElementFields( region, ug->GetCellData() );
-
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
@@ -1109,7 +1108,7 @@ void VTKPolyDataWriterInterface::writeSurfaceElementRegions( real64 const time,
                                                              NodeManager const & nodeManager,
                                                              EmbeddedSurfaceNodeManager const & embSurfNodeManager,
                                                              FaceManager const & faceManager,
-                                                             string const & path ) const
+                                                             string const & path )
 {
   elemManager.forElementRegions< SurfaceElementRegion >( [&]( SurfaceElementRegion const & region )
   {
@@ -1140,9 +1139,7 @@ void VTKPolyDataWriterInterface::writeSurfaceElementRegions( real64 const time,
 
     writeTimestamp( ug.GetPointer(), time );
     writeElementFields( region, ug->GetCellData() );
-
-    string const regionDir = joinPath( path, region.getName() );
-    writeUnstructuredGrid( regionDir, ug.GetPointer() );
+    writeUnstructuredGrid( path, region, ug.GetPointer() );
   } );
 }
 
@@ -1188,13 +1185,13 @@ void VTKPolyDataWriterInterface::writeVtmFile( integer const cycle,
 
       string const meshPath = joinPath( getCycleSubFolder( cycle ), meshBodyName, meshLevelName );
 
-      int const mpiSize = MpiWrapper::commSize();
+      // int const mpiSize = MpiWrapper::commSize();
 
       auto addElementRegion = [&]( ElementRegionBase const & region )
       {
         std::vector< string > const blockPath{ meshBody.getName(), meshLevel.getName(), region.getCatalogName(), region.getName() };
         string const regionPath = joinPath( meshPath, region.getName() );
-        for( int i = 0; i < mpiSize; i++ )
+        for( const auto & i : m_targetProcessesId.at( region.getName()) )
         {
           string const dataSetName = getRankFileName( i );
           string const dataSetFile = joinPath( regionPath, dataSetName + ".vtu" );
@@ -1207,7 +1204,7 @@ void VTKPolyDataWriterInterface::writeVtmFile( integer const cycle,
         string const & regionName = region.getName();
         std::vector< string > const blockPath{ meshBodyName, meshLevelName, region.getCatalogName(), regionName };
         string const regionPath = joinPath( meshPath, regionName );
-        for( int i = 0; i < mpiSize; i++ )
+        for( const auto & i : m_targetProcessesId.at( region.getName()) )
         {
           string const dataSetName = getRankFileName( i );
           string const dataSetFile = joinPath( regionPath, dataSetName + ".vtu" );
@@ -1256,8 +1253,11 @@ int toVtkOutputMode( VTKOutputMode const mode )
 }
 
 void VTKPolyDataWriterInterface::writeUnstructuredGrid( string const & path,
-                                                        vtkUnstructuredGrid * ug ) const
+                                                        ObjectManagerBase const & region,
+                                                        vtkUnstructuredGrid * ug )
 {
+  string const regionDir = joinPath( path, region.getName() );
+
   vtkSmartPointer< vtkAlgorithm > filter;
 
   // If we want to get rid of the ghost ranks, we use the appropriate `vtkThreshold` filter.
@@ -1279,15 +1279,51 @@ void VTKPolyDataWriterInterface::writeUnstructuredGrid( string const & path,
   }
 
   filter->SetInputDataObject( ug );
-  filter->Update();
-
-  makeDirectory( path );
-  string const vtuFilePath = joinPath( path, getRankFileName( MpiWrapper::commRank() ) + ".vtu" );
-  auto const vtuWriter = vtkSmartPointer< vtkXMLUnstructuredGridWriter >::New();
-  vtuWriter->SetInputData( filter->GetOutputDataObject( 0 ) );
-  vtuWriter->SetFileName( vtuFilePath.c_str() );
-  vtuWriter->SetDataMode( toVtkOutputMode( m_outputMode ) );
-  vtuWriter->Write();
+
+  vtkSmartPointer< vtkMultiProcessController > controller = vtk::getController();
+  vtkMultiProcessController::SetGlobalController( controller );
+
+  // In case of m_numberOfTargetProcesses == GetNumberOfProcesses the filter returns a shallow copy
+  // The behavior  is the same as previously in this case. The rank number is computed instead of implicitly written
+  vtkNew< vtkAggregateDataSetFilter > aggregate;
+  aggregate->SetInputConnection( filter->GetOutputPort());
+  aggregate->SetNumberOfTargetProcesses( m_numberOfTargetProcesses );
+  aggregate->SetMergePoints( false );
+  aggregate->Update();
+
+  int localCommRank = -1;
+  if( vtkDataSet::SafeDownCast( aggregate->GetOutput())->GetNumberOfPoints() != 0 )
+  {
+    localCommRank = MpiWrapper::commRank();
+    makeDirectory( regionDir );
+    string const vtuFilePath = joinPath( regionDir, getRankFileName( localCommRank ) + ".vtu" );
+    auto const vtuWriter = vtkSmartPointer< vtkXMLUnstructuredGridWriter >::New();
+    vtuWriter->SetInputData( aggregate->GetOutput() );
+    vtuWriter->SetFileName( vtuFilePath.c_str() );
+    vtuWriter->SetDataMode( toVtkOutputMode( m_outputMode ) );
+    vtuWriter->Write();
+  }
+
+  const int size = MpiWrapper::commSize( MPI_COMM_GEOS );
+  std::vector< int > globalValues( size );
+
+  // Everything is done on rank 0
+  MpiWrapper::gather( &localCommRank,
+                      1,
+                      globalValues.data(),
+                      1,
+                      0,
+                      MPI_COMM_GEOS );
+
+  if( MpiWrapper::commRank() == 0 )
+  {
+    // any rank that does not hold data will not participate in the output
+    globalValues.erase( std::remove_if( globalValues.begin(),
+                                        globalValues.end(),
+                                        []( int x ) { return x == -1; } ),
+                        globalValues.end());
+    m_targetProcessesId[region.getName()] = globalValues;
+  }
 }
 
 void VTKPolyDataWriterInterface::write( real64 const time,