Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BUG: VTK Rectilinear Grid Writer Bufferring #765

Merged
merged 4 commits into from
Nov 13, 2023
Merged

BUG: VTK Rectilinear Grid Writer Bufferring #765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c96cdf0
buffer dump every 2mb
nyoungbq Nov 9, 2023
6e3d9d9
clean up duplication bug
nyoungbq Nov 9, 2023
7f84cc9
Update algorithm to buffer with a std::string with a reserved size
imikejackson Nov 10, 2023
f032a92
Fix failing unite test with updated exemplar file.
imikejackson Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/VtkRectilinearGridWriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ using namespace complex;

namespace
{
static constexpr usize k_BufferDumpVal = 1000000;

// -----------------------------------------------------------------------------
template <typename T>
std::string TypeForPrimitive(T value, const IFilter::MessageHandler& messageHandler)
Expand Down Expand Up @@ -177,24 +179,41 @@ struct WriteVtkDataArrayFunctor
}
else
{
std::stringstream ss;
std::string buffer;
buffer.reserve(k_BufferDumpVal);
for(size_t i = 0; i < totalElements; i++)
{
if(i % 20 == 0 && i > 0)
{
ss << "\n";
buffer.append("\n");
}
if(useIntCast)
{
ss << " " << static_cast<int>(dataArray[i]);
buffer.append(fmt::format(" {:d}", static_cast<int>(dataArray[i])));
}
else if constexpr(std::is_same_v<T, float32>)
{
buffer.append(fmt::format(" {:f}", dataArray[i]));
}
else if constexpr(std::is_same_v<T, float64>)
{
buffer.append(fmt::format(" {:f}", dataArray[i]));
}
else
{
ss << " " << dataArray[i];
buffer.append(fmt::format(" {}", dataArray[i]));
}
// If the buffer is within 32 bytes of the reserved size, then dump
// the contents to the file.
if(buffer.size() > (k_BufferDumpVal - 32))
{
fprintf(outputFile, "%s", buffer.c_str());
buffer.clear();
buffer.reserve(k_BufferDumpVal);
}
}
ss << "\n";
fprintf(outputFile, "%s", ss.str().c_str());
buffer.append("\n");
fprintf(outputFile, "%s", buffer.c_str());
}
}
};
Expand Down
2 changes: 1 addition & 1 deletion src/Plugins/ComplexCore/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ if(EXISTS "${DREAM3D_DATA_DIR}" AND COMPLEX_DOWNLOAD_TEST_FILES)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_resample_rect_grid_to_image_geom.tar.gz SHA512 58cabfb83deeef7aff1252963a8cb378745ca00fc120a9623f699b4c9c618d216c4d3fa88e964772706a0096e979abc741be7ba6f96a2398162b3f6d4ec8ec13)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_combine_stl_files.tar.gz SHA512 109dfb39ac04213a7d8bc8ef3d852f112177ad1a1c624926e980c2a3e30af04ff3c48b8cf315d04fd2b8d0270b1defa33078511243d9ac2def37a8b43884120b)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_avizo_writers.tar.gz SHA512 f776eb5c86e06bde0fb1ee76dbdf95e4fd1457697033b2c639cac376db3ba0b05410ed4074fb10a47dd26ef79c78a02b5bb77c04cfe1299e8a33d8b3bff09749)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_vtk_rectilinear_grid_writer.tar.gz SHA512 a9ebe208ac5ab6d0df4f416130e54ac5642f7f57b43a025d093b7bc1c8e680c69501fc32c079e38ed4ba4197d07352124fe317e11f3c517311ae83867aca0c44 )
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME vtk_rectilinear_grid_writer.tar.gz SHA512 9fef02b5269609503d03dd0126cc635cc1c1156894cff18b0184b334d705b850ca1a06ae0d1c66a352a32dd9ad9fb74f24255c7de1399b06bbec7d2e2b41941b )
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME Small_IN100_dream3d.tar.gz SHA512 6dd8a3412532bdc7481f7781c7087b4477c6a1efbe6b214f997dad30c53c59714a751be522f084b98065fe75100c74df901bb8af2f512ef47344d8f7941575cf )
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME 6_6_sample_surface_mesh.tar.gz SHA512 b1eec1127d125303a4757a482a7fd4b6c0e42241f01f8afabec9a15dc1ffa6d732afc11b4623aaf81c7a9e53de59423ef4bbc470710c31fe9a5c61916bdf544e)
download_test_data(DREAM3D_DATA_DIR ${DREAM3D_DATA_DIR} ARCHIVE_NAME bin_feature_phases.tar.gz SHA512 e43805c5a69d08ce6d657f8fa3fb9fec7d174f2329412fa62ba95ac6d82f1983ed2c5a641d81651ba86e6eeb5d20602b6e131f34094160305c4bf03cfbd06be1)
Expand Down
17 changes: 10 additions & 7 deletions src/Plugins/ComplexCore/test/VtkRectilinearGridWriterTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution",
{
const complex::UnitTest::TestFileSentinel testDataSentinel(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "Small_IN100_dream3d.tar.gz", "Small_IN100.dream3d");

const complex::UnitTest::TestFileSentinel testDataSentinel1(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "6_6_vtk_rectilinear_grid_writer.tar.gz",
"6_6_vtk_rectilinear_grid_writer");

const complex::UnitTest::TestFileSentinel testDataSentinel1(complex::unit_test::k_CMakeExecutable, complex::unit_test::k_TestFilesDir, "vtk_rectilinear_grid_writer.tar.gz",
"vtk_rectilinear_grid_writer");
// Read input DREAM3D File
auto exemplarFilePath = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir));
DataStructure dataStructure = UnitTest::LoadDataStructure(exemplarFilePath);
Expand All @@ -28,10 +27,10 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution",
VtkRectilinearGridWriterFilter filter;
Arguments args;

fs::path exemplarOutputPath = fs::path(fmt::format("{}/6_6_vtk_rectilinear_grid_writer/6_6_vtk_rectilinear_grid.vtk", unit_test::k_TestFilesDir));
fs::path computedOutputPath(fmt::format("{}/NX_vtk_rectilinear_grid.vtk", unit_test::k_BinaryTestOutputDir));
fs::path exemplarBinaryOutputPath = fs::path(fmt::format("{}/6_6_vtk_rectilinear_grid_writer/6_6_vtk_rectilinear_grid_binary.vtk", unit_test::k_TestFilesDir));
fs::path computedBinaryOutputPath(fmt::format("{}/NX_vtk_rectilinear_grid_binary.vtk", unit_test::k_BinaryTestOutputDir));
fs::path exemplarOutputPath = fs::path(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid.vtk", unit_test::k_TestFilesDir));
fs::path computedOutputPath(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid.vtk", unit_test::k_BinaryTestOutputDir));
fs::path exemplarBinaryOutputPath = fs::path(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid_binary.vtk", unit_test::k_TestFilesDir));
fs::path computedBinaryOutputPath(fmt::format("{}/vtk_rectilinear_grid_writer/vtk_rectilinear_grid_binary.vtk", unit_test::k_BinaryTestOutputDir));

MultiArraySelectionParameter::ValueType selectedArrayPaths = {k_ConfidenceIndexArrayPath, k_EulersArrayPath, k_FitArrayPath, k_ImageQualityArrayPath, k_PhasesArrayPath, k_SEMSignalArrayPath};

Expand Down Expand Up @@ -64,6 +63,10 @@ TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: Valid Filter Execution",
std::ifstream computedBinaryFile(computedBinaryOutputPath, std::ios::binary);
std::ifstream exemplarBinaryFile(exemplarBinaryOutputPath, std::ios::binary);
UnitTest::CompareAsciiFiles(computedBinaryFile, exemplarBinaryFile, linesToSkip);

// Remove the test files since they can get quite large.
std::error_code errorCode;
std::filesystem::remove_all(fmt::format("{}/vtk_rectilinear_grid_writer", unit_test::k_BinaryTestOutputDir), errorCode);
}

TEST_CASE("ComplexCore::VtkRectilinearGridWriterFilter: InValid Filter Execution", "[ComplexCore][VtkRectilinearGridWriterFilter]")
Expand Down
Loading