Skip to content

Commit

Permalink
BUG: VTK Rectilinear Grid Writer Bufferring (#765)
Browse files Browse the repository at this point in the history 
* buffer dump every 2mb

* clean up duplication bug
  • Loading branch information
@nyoungbq
nyoungbq authored Nov 13, 2023
1 parent 63c80dc commit 461f0a2
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 14 deletions.
31 changes: 25 additions & 6 deletions src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ using namespace complex;

namespace
{
static constexpr usize k_BufferDumpVal = 1000000;

// -----------------------------------------------------------------------------
template <typename T>
std::string TypeForPrimitive(T value, const IFilter::MessageHandler& messageHandler)
Expand Down Expand Up @@ -177,24 +179,41 @@ struct WriteVtkDataArrayFunctor
}
else
{
std::stringstream ss;
std::string buffer;
buffer.reserve(k_BufferDumpVal);
for(size_t i = 0; i < totalElements; i++)
{
if(i % 20 == 0 && i > 0)
{
ss << "\n";
buffer.append("\n");
}
if(useIntCast)
{
ss << " " << static_cast<int>(dataArray[i]);
buffer.append(fmt::format(" {:d}", static_cast<int>(dataArray[i])));
}
else if constexpr(std::is_same_v<T, float32>)
{
buffer.append(fmt::format(" {:f}", dataArray[i]));
}
else if constexpr(std::is_same_v<T, float64>)
{
buffer.append(fmt::format(" {:f}", dataArray[i]));
}
else
{
ss << " " << dataArray[i];
buffer.append(fmt::format(" {}", dataArray[i]));
}
// If the buffer is within 32 bytes of the reserved size, then dump
// the contents to the file.
if(buffer.size() > (k_BufferDumpVal - 32))
{
fprintf(outputFile, "%s", buffer.c_str());
buffer.clear();
buffer.reserve(k_BufferDumpVal);
}
}
ss << "\n";
fprintf(outputFile, "%s", ss.str().c_str());
buffer.append("\n");
fprintf(outputFile, "%s", buffer.c_str());
}
}
};
Expand Down
2 changes: 1 addition & 1 deletion src/Plugins/ComplexCore/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ if(EXISTS "${DREAM3D_DATA_DIR}" AND COMPLEX_DOWNLOAD_TEST_FILES)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_resample_rect_grid_to_image_geom.tar.gz SHA512 58cabfb83deeef7aff1252963a8cb378745ca00fc120a9623f699b4c9c618d216c4d3fa88e964772706a0096e979abc741be7ba6f96a2398162b3f6d4ec8ec13)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_combine_stl_files.tar.gz SHA512 109dfb39ac04213a7d8bc8ef3d852f112177ad1a1c624926e980c2a3e30af04ff3c48b8cf315d04fd2b8d0270b1defa33078511243d9ac2def37a8b43884120b)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_avizo_writers.tar.gz SHA512 f776eb5c86e06bde0fb1ee76dbdf95e4fd1457697033b2c639cac376db3ba0b05410ed4074fb10a47dd26ef79c78a02b5bb77c04cfe1299e8a33d8b3bff09749)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_vtk_rectilinear_grid_writer.tar.gz SHA512 a9ebe208ac5ab6d0df4f416130e54ac5642f7f57b43a025d093b7bc1c8e680c69501fc32c079e38ed4ba4197d07352124fe317e11f3c517311ae83867aca0c44 )
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME vtk_rectilinear_grid_writer.tar.gz SHA512 9fef02b5269609503d03dd0126cc635cc1c1156894cff18b0184b334d705b850ca1a06ae0d1c66a352a32dd9ad9fb74f24255c7de1399b06bbec7d2e2b41941b )
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME Small_IN100_dream3d.tar.gz SHA512 6dd8a3412532bdc7481f7781c7087b4477c6a1efbe6b214f997dad30c53c59714a751be522f084b98065fe75100c74df901bb8af2f512ef47344d8f7941575cf )
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_sample_surface_mesh.tar.gz SHA512 b1eec1127d125303a4757a482a7fd4b6c0e42241f01f8afabec9a15dc1ffa6d732afc11b4623aaf81c7a9e53de59423ef4bbc470710c31fe9a5c61916bdf544e)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME bin_feature_phases.tar.gz SHA512 e43805c5a69d08ce6d657f8fa3fb9fec7d174f2329412fa62ba95ac6d82f1983ed2c5a641d81651ba86e6eeb5d20602b6e131f34094160305c4bf03cfbd06be1)
Expand Down
17 changes: 10 additions & 7 deletions src/Plugins/ComplexCore/test/VtkRectilinearGridWriterTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution",
{
const complex::UnitTest::TestFileSentinel testDataSentinel(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "Small_IN100_dream3d.tar.gz", "Small_IN100.dream3d");

const complex::UnitTest::TestFileSentinel testDataSentinel1(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "6_6_vtk_rectilinear_grid_writer.tar.gz",
"6_6_vtk_rectilinear_grid_writer");

const complex::UnitTest::TestFileSentinel testDataSentinel1(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "vtk_rectilinear_grid_writer.tar.gz",
"vtk_rectilinear_grid_writer");
// Read input DREAM3D File
auto exemplarFilePath = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir));
DataStructure dataStructure = UnitTest::LoadDataStructure(exemplarFilePath);
Expand All @@ -28,10 +27,10 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution",
VtkRectilinearGridWriterFilter filter;
Arguments args;

fs::path exemplarOutputPath = fs::path(fmt::format("{}/6_6_vtk_rectilinear_grid_writer/6_6_vtk_rectilinear_grid.vtk", unit_test::k_TestFilesDir));
fs::path computedOutputPath(fmt::format("{}/NX_vtk_rectilinear_grid.vtk", unit_test::k_BinaryTestOutputDir));
fs::path exemplarBinaryOutputPath = fs::path(fmt::format("{}/6_6_vtk_rectilinear_grid_writer/6_6_vtk_rectilinear_grid_binary.vtk", unit_test::k_TestFilesDir));
fs::path computedBinaryOutputPath(fmt::format("{}/NX_vtk_rectilinear_grid_binary.vtk", unit_test::k_BinaryTestOutputDir));
fs::path exemplarOutputPath = fs::path(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid.vtk", unit_test::k_TestFilesDir));
fs::path computedOutputPath(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid.vtk", unit_test::k_BinaryTestOutputDir));
fs::path exemplarBinaryOutputPath = fs::path(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid_binary.vtk", unit_test::k_TestFilesDir));
fs::path computedBinaryOutputPath(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid_binary.vtk", unit_test::k_BinaryTestOutputDir));

MultiArraySelectionParameter::ValueType selectedArrayPaths = {k_ConfidenceIndexArrayPath, k_EulersArrayPath, k_FitArrayPath, k_ImageQualityArrayPath, k_PhasesArrayPath, k_SEMSignalArrayPath};

Expand Down Expand Up @@ -64,6 +63,10 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution",
std::ifstream computedBinaryFile(computedBinaryOutputPath, std::ios::binary);
std::ifstream exemplarBinaryFile(exemplarBinaryOutputPath, std::ios::binary);
UnitTest::CompareAsciiFiles(computedBinaryFile, exemplarBinaryFile, linesToSkip);

// Remove the test files since they can get quite large.
std::error_code errorCode;
std::filesystem::remove_all(fmt::format("{}/vtk_rectilinear_grid_writer", unit_test::k_BinaryTestOutputDir), errorCode);
}

TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: InValid Filter Execution", "[ComplexCore][VtkRectilinearGridWriterFilter]")
Expand Down

0 comments on commit 461f0a2

Please sign in to comment.