From c96cdf05f3e8192ee2748b95ec21169933bc1639 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Thu, 9 Nov 2023 15:16:36 -0500 Subject: [PATCH 1/4] buffer dump every 2mb --- .../Filters/Algorithms/VtkRectilinearGridWriter.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp index b4461a7cc5..4de9210299 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp @@ -11,6 +11,8 @@ using namespace complex; namespace { +static constexpr usize k_BufferDumpVal = 2000000; + // ----------------------------------------------------------------------------- template std::string TypeForPrimitive(T value, const IFilter::MessageHandler& messageHandler) @@ -178,6 +180,7 @@ struct WriteVtkDataArrayFunctor else { std::stringstream ss; + usize nextDump = k_BufferDumpVal; for(size_t i = 0; i < totalElements; i++) { if(i % 20 == 0 && i > 0) @@ -192,6 +195,14 @@ struct WriteVtkDataArrayFunctor { ss << " " << dataArray[i]; } + + if(i > nextDump) + { + fprintf(outputFile, "%s", ss.str().c_str()); + ss.clear(); + + nextDump += k_BufferDumpVal; + } } ss << "\n"; fprintf(outputFile, "%s", ss.str().c_str()); From 6e3d9d9235025689ab5183bafc7dc348bfaf8239 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Thu, 9 Nov 2023 16:43:27 -0500 Subject: [PATCH 2/4] clean up duplication bug --- .../Algorithms/VtkRectilinearGridWriter.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp index 4de9210299..5aafdb67df 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp @@ -186,6 +186,15 @@ struct WriteVtkDataArrayFunctor if(i % 20 == 0 && i > 0) { ss << "\n"; + + if(i > nextDump) + { + fprintf(outputFile, "%s", ss.str().c_str()); + + ss = std::stringstream(); + + nextDump += k_BufferDumpVal; + } } if(useIntCast) { @@ -195,14 +204,6 @@ struct WriteVtkDataArrayFunctor { ss << " " << dataArray[i]; } - - if(i > nextDump) - { - fprintf(outputFile, "%s", ss.str().c_str()); - ss.clear(); - - nextDump += k_BufferDumpVal; - } } ss << "\n"; fprintf(outputFile, "%s", ss.str().c_str()); From 7f84cc94bc480a2ef5bcbba06bb2c0f2e73ae7cb Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Fri, 10 Nov 2023 15:15:25 -0500 Subject: [PATCH 3/4] Update algorithm to buffer with a std::string with a reserved size This will require a new test file Signed-off-by: Michael Jackson --- .../Algorithms/VtkRectilinearGridWriter.cpp | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp index 5aafdb67df..b065646142 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp @@ -11,7 +11,7 @@ using namespace complex; namespace { -static constexpr usize k_BufferDumpVal = 2000000; +static constexpr usize k_BufferDumpVal = 1000000; // ----------------------------------------------------------------------------- template @@ -179,34 +179,41 @@ struct WriteVtkDataArrayFunctor } else { - std::stringstream ss; - usize nextDump = k_BufferDumpVal; + std::string buffer; + buffer.reserve(k_BufferDumpVal); for(size_t i = 0; i < totalElements; i++) { if(i % 20 == 0 && i > 0) { - ss << "\n"; - - if(i > nextDump) - { - fprintf(outputFile, "%s", ss.str().c_str()); - - ss = std::stringstream(); - - nextDump += k_BufferDumpVal; - } + buffer.append("\n"); } if(useIntCast) { - ss << " " << static_cast(dataArray[i]); + buffer.append(fmt::format(" {:d}", static_cast(dataArray[i]))); + } + else if constexpr(std::is_same_v) + { + buffer.append(fmt::format(" {:f}", dataArray[i])); + } + else if constexpr(std::is_same_v) + { + buffer.append(fmt::format(" {:f}", dataArray[i])); } else { - ss << " " << dataArray[i]; + buffer.append(fmt::format(" {}", dataArray[i])); + } + // If the buffer is within 32 bytes of the reserved size, then dump + // the contents to the file. + if(buffer.size() > (k_BufferDumpVal - 32)) + { + fprintf(outputFile, "%s", buffer.c_str()); + buffer.clear(); + buffer.reserve(k_BufferDumpVal); } } - ss << "\n"; - fprintf(outputFile, "%s", ss.str().c_str()); + buffer.append("\n"); + fprintf(outputFile, "%s", buffer.c_str()); } } }; From f032a92d7d9b3eda7d7e932e2edafaea13a93ab9 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Mon, 13 Nov 2023 11:02:54 -0500 Subject: [PATCH 4/4] Fix failing unite test with updated exemplar file. Signed-off-by: Michael Jackson --- src/Plugins/ComplexCore/test/CMakeLists.txt | 2 +- .../test/VtkRectilinearGridWriterTest.cpp | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Plugins/ComplexCore/test/CMakeLists.txt b/src/Plugins/ComplexCore/test/CMakeLists.txt index 3d7c1fc089..a8d333d714 100644 --- a/src/Plugins/ComplexCore/test/CMakeLists.txt +++ b/src/Plugins/ComplexCore/test/CMakeLists.txt @@ -207,7 +207,7 @@ if(EXISTS "${DREAM3D_DATA_DIR}" AND COMPLEX_DOWNLOAD_TEST_FILES) download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_resample_rect_grid_to_image_geom.tar.gz SHA512 58cabfb83deeef7aff1252963a8cb378745ca00fc120a9623f699b4c9c618d216c4d3fa88e964772706a0096e979abc741be7ba6f96a2398162b3f6d4ec8ec13) download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_combine_stl_files.tar.gz SHA512 109dfb39ac04213a7d8bc8ef3d852f112177ad1a1c624926e980c2a3e30af04ff3c48b8cf315d04fd2b8d0270b1defa33078511243d9ac2def37a8b43884120b) download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_avizo_writers.tar.gz SHA512 f776eb5c86e06bde0fb1ee76dbdf95e4fd1457697033b2c639cac376db3ba0b05410ed4074fb10a47dd26ef79c78a02b5bb77c04cfe1299e8a33d8b3bff09749) - download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_vtk_rectilinear_grid_writer.tar.gz SHA512 a9ebe208ac5ab6d0df4f416130e54ac5642f7f57b43a025d093b7bc1c8e680c69501fc32c079e38ed4ba4197d07352124fe317e11f3c517311ae83867aca0c44 ) + download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME vtk_rectilinear_grid_writer.tar.gz SHA512 9fef02b5269609503d03dd0126cc635cc1c1156894cff18b0184b334d705b850ca1a06ae0d1c66a352a32dd9ad9fb74f24255c7de1399b06bbec7d2e2b41941b ) download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME Small_IN100_dream3d.tar.gz SHA512 6dd8a3412532bdc7481f7781c7087b4477c6a1efbe6b214f997dad30c53c59714a751be522f084b98065fe75100c74df901bb8af2f512ef47344d8f7941575cf ) download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_sample_surface_mesh.tar.gz SHA512 b1eec1127d125303a4757a482a7fd4b6c0e42241f01f8afabec9a15dc1ffa6d732afc11b4623aaf81c7a9e53de59423ef4bbc470710c31fe9a5c61916bdf544e) download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME bin_feature_phases.tar.gz SHA512 e43805c5a69d08ce6d657f8fa3fb9fec7d174f2329412fa62ba95ac6d82f1983ed2c5a641d81651ba86e6eeb5d20602b6e131f34094160305c4bf03cfbd06be1) diff --git a/src/Plugins/ComplexCore/test/VtkRectilinearGridWriterTest.cpp b/src/Plugins/ComplexCore/test/VtkRectilinearGridWriterTest.cpp index dcad2b8ff5..9761a726db 100644 --- a/src/Plugins/ComplexCore/test/VtkRectilinearGridWriterTest.cpp +++ b/src/Plugins/ComplexCore/test/VtkRectilinearGridWriterTest.cpp @@ -17,9 +17,8 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution", { const complex::UnitTest::TestFileSentinel testDataSentinel(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "Small_IN100_dream3d.tar.gz", "Small_IN100.dream3d"); - const complex::UnitTest::TestFileSentinel testDataSentinel1(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "6_6_vtk_rectilinear_grid_writer.tar.gz", - "6_6_vtk_rectilinear_grid_writer"); - + const complex::UnitTest::TestFileSentinel testDataSentinel1(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "vtk_rectilinear_grid_writer.tar.gz", + "vtk_rectilinear_grid_writer"); // Read input DREAM3D File auto exemplarFilePath = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir)); DataStructure dataStructure = UnitTest::LoadDataStructure(exemplarFilePath); @@ -28,10 +27,10 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution", VtkRectilinearGridWriterFilter filter; Arguments args; - fs::path exemplarOutputPath = fs::path(fmt::format("{}/6_6_vtk_rectilinear_grid_writer/6_6_vtk_rectilinear_grid.vtk", unit_test::k_TestFilesDir)); - fs::path computedOutputPath(fmt::format("{}/NX_vtk_rectilinear_grid.vtk", unit_test::k_BinaryTestOutputDir)); - fs::path exemplarBinaryOutputPath = fs::path(fmt::format("{}/6_6_vtk_rectilinear_grid_writer/6_6_vtk_rectilinear_grid_binary.vtk", unit_test::k_TestFilesDir)); - fs::path computedBinaryOutputPath(fmt::format("{}/NX_vtk_rectilinear_grid_binary.vtk", unit_test::k_BinaryTestOutputDir)); + fs::path exemplarOutputPath = fs::path(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid.vtk", unit_test::k_TestFilesDir)); + fs::path computedOutputPath(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid.vtk", unit_test::k_BinaryTestOutputDir)); + fs::path exemplarBinaryOutputPath = fs::path(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid_binary.vtk", unit_test::k_TestFilesDir)); + fs::path computedBinaryOutputPath(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid_binary.vtk", unit_test::k_BinaryTestOutputDir)); MultiArraySelectionParameter::ValueType selectedArrayPaths = {k_ConfidenceIndexArrayPath, k_EulersArrayPath, k_FitArrayPath, k_ImageQualityArrayPath, k_PhasesArrayPath, k_SEMSignalArrayPath}; @@ -64,6 +63,10 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution", std::ifstream computedBinaryFile(computedBinaryOutputPath, std::ios::binary); std::ifstream exemplarBinaryFile(exemplarBinaryOutputPath, std::ios::binary); UnitTest::CompareAsciiFiles(computedBinaryFile, exemplarBinaryFile, linesToSkip); + + // Remove the test files since they can get quite large. + std::error_code errorCode; + std::filesystem::remove_all(fmt::format("{}/vtk_rectilinear_grid_writer", unit_test::k_BinaryTestOutputDir), errorCode); } TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: InValid Filter Execution", "[ComplexCore][VtkRectilinearGridWriterFilter]")