From ee87be8f2511e88c23c381422b22aed6c6374efb Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Wed, 14 Feb 2024 14:00:07 -0800 Subject: [PATCH 01/29] Initial pass using whole frame compression --- cmake/OpenEXRSetup.cmake | 21 ++ src/lib/OpenEXR/CMakeLists.txt | 4 +- src/lib/OpenEXR/ImfCompression.h | 2 + src/lib/OpenEXR/ImfCompressionAttribute.cpp | 2 +- src/lib/OpenEXR/ImfCompressor.cpp | 9 +- src/lib/OpenEXR/ImfHTCompressor.cpp | 210 +++++++++++ src/lib/OpenEXR/ImfHTCompressor.h | 83 +++++ src/test/OpenEXRTest/CMakeLists.txt | 2 + src/test/OpenEXRTest/main.cpp | 2 + src/test/OpenEXRTest/testHTCompression.cpp | 366 ++++++++++++++++++++ src/test/OpenEXRTest/testHTCompression.h | 8 + 11 files changed, 706 insertions(+), 3 deletions(-) create mode 100644 src/lib/OpenEXR/ImfHTCompressor.cpp create mode 100644 src/lib/OpenEXR/ImfHTCompressor.h create mode 100644 src/test/OpenEXRTest/testHTCompression.cpp create mode 100644 src/test/OpenEXRTest/testHTCompression.h diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index a70f86d2..3022ec50 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -240,6 +240,27 @@ else() set(EXR_DEFLATE_LIB) endif() + +####################################### +# Install OpenJPH +####################################### + +# message(STATUS "Fetching OpenJPH") + +# include(FetchContent) +# FetchContent_Declare( +# openjph +# GIT_REPOSITORY https://github.com/aous72/OpenJPH +# GIT_TAG origin/master +# ) + +# FetchContent_MakeAvailable(openjph) + +find_path(OJPH_INCLUDE_DIR ojph_file.h PATH_SUFFIXES openjph) +include_directories(${OJPH_INCLUDE_DIR}) +find_library(OJPH_LIBRARY NAMES openjph) + + ####################################### # Find or install Imath ####################################### diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index b58e0bac..5ac1a978 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -1,6 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) Contributors to the OpenEXR Project. - openexr_define_library(OpenEXR PRIV_EXPORT OPENEXR_EXPORTS CURDIR ${CMAKE_CURRENT_SOURCE_DIR} @@ -67,6 +66,7 @@ openexr_define_library(OpenEXR ImfGenericInputFile.cpp ImfGenericOutputFile.cpp ImfHeader.cpp + ImfHTCompressor.cpp ImfHuf.cpp ImfIDManifest.cpp ImfIDManifestAttribute.cpp @@ -161,6 +161,7 @@ openexr_define_library(OpenEXR ImfGenericInputFile.h ImfGenericOutputFile.h ImfHeader.h + ImfHTCompressor.h ImfHuf.h ImfIDManifest.h ImfIDManifestAttribute.h @@ -219,4 +220,5 @@ openexr_define_library(OpenEXR OpenEXR::Iex OpenEXR::IlmThread OpenEXR::OpenEXRCore + ${OJPH_LIBRARY} ) diff --git a/src/lib/OpenEXR/ImfCompression.h b/src/lib/OpenEXR/ImfCompression.h index fbc0fa24..868b9951 100644 --- a/src/lib/OpenEXR/ImfCompression.h +++ b/src/lib/OpenEXR/ImfCompression.h @@ -45,6 +45,8 @@ enum IMF_EXPORT_ENUM Compression // wise and faster to decode full frames // than DWAA_COMPRESSION. + HT_COMPRESSION = 10, + NUM_COMPRESSION_METHODS // number of different compression methods }; diff --git a/src/lib/OpenEXR/ImfCompressionAttribute.cpp b/src/lib/OpenEXR/ImfCompressionAttribute.cpp index b0e93db4..f11ed11d 100644 --- a/src/lib/OpenEXR/ImfCompressionAttribute.cpp +++ b/src/lib/OpenEXR/ImfCompressionAttribute.cpp @@ -57,7 +57,7 @@ CompressionAttribute::readValueFrom ( tmp != ZIPS_COMPRESSION && tmp != ZIP_COMPRESSION && tmp != PIZ_COMPRESSION && tmp != PXR24_COMPRESSION && tmp != B44_COMPRESSION && tmp != B44A_COMPRESSION && - tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION) + tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION && tmp != HT_COMPRESSION) { tmp = NUM_COMPRESSION_METHODS; } diff --git a/src/lib/OpenEXR/ImfCompressor.cpp b/src/lib/OpenEXR/ImfCompressor.cpp index 4d9e99a3..ff5905fc 100644 --- a/src/lib/OpenEXR/ImfCompressor.cpp +++ b/src/lib/OpenEXR/ImfCompressor.cpp @@ -18,6 +18,7 @@ #include "ImfPxr24Compressor.h" #include "ImfRleCompressor.h" #include "ImfZipCompressor.h" +#include "ImfHTCompressor.h" OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -63,7 +64,8 @@ isValidCompression (Compression c) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: - case DWAB_COMPRESSION: return true; + case DWAB_COMPRESSION: + case HT_COMPRESSION: return true; default: return false; } @@ -141,6 +143,10 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr) 256, DwaCompressor::STATIC_HUFFMAN); + case HT_COMPRESSION: + + return new HTCompressor (hdr); + default: return 0; } } @@ -163,6 +169,7 @@ numLinesInBuffer (Compression comp) case B44A_COMPRESSION: case DWAA_COMPRESSION: return 32; case DWAB_COMPRESSION: return 256; + case HT_COMPRESSION: return 16000; default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type"); } diff --git a/src/lib/OpenEXR/ImfHTCompressor.cpp b/src/lib/OpenEXR/ImfHTCompressor.cpp new file mode 100644 index 00000000..d8f3aaa5 --- /dev/null +++ b/src/lib/OpenEXR/ImfHTCompressor.cpp @@ -0,0 +1,210 @@ +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + +//----------------------------------------------------------------------------- +// +// class HTCompressor +// +//----------------------------------------------------------------------------- + +#include "ImfHTCompressor.h" +#include "ImfAutoArray.h" +#include "ImfChannelList.h" +#include "ImfCheckedArithmetic.h" +#include "ImfHeader.h" +#include "ImfIO.h" +#include "ImfMisc.h" +#include "ImfNamespace.h" +#include "ImfXdr.h" +#include +#include + +#include + +#include +#include +#include +#include + +using IMATH_NAMESPACE::Box2i; + +OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER + +HTCompressor::HTCompressor (const Header& hdr) + : Compressor (hdr), _num_comps (0), _buffer (NULL) +{ + const ChannelList& channels = header ().channels (); + + for (ChannelList::ConstIterator c = channels.begin (); c != channels.end (); + ++c) + { + assert (c.channel ().type == HALF); + assert (c.channel ().xSampling == 1); + assert (c.channel ().ySampling == 1); + this->_num_comps++; + } +} + +HTCompressor::~HTCompressor () +{ + this->_output.close (); + this->_input.close (); + delete[] this->_buffer; +} + +int +HTCompressor::numScanLines () const +{ + return 16000; +} + +Compressor::Format +HTCompressor::format () const +{ + return Compressor::Format::NATIVE; +} + +int +HTCompressor::compress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) +{ + Box2i dw = this->header ().dataWindow (); + ojph::ui32 height = dw.size ().y + 1; + ojph::ui32 width = dw.size ().x + 1; + + this->_codestream.set_planar (false); + + ojph::param_siz siz = this->_codestream.access_siz (); + + siz.set_num_components (this->_num_comps); + for (ojph::ui32 c = 0; c < this->_num_comps; c++) + siz.set_component (c, ojph::point (1, 1), 16, false); + + /* TODO: extend to multiple tiles and non-coincident data and display windows */ + siz.set_image_offset (ojph::point (0, 0)); + siz.set_tile_offset (ojph::point (0, 0)); + siz.set_image_extent (ojph::point (width, height)); + siz.set_tile_size (ojph::size (width, height)); + + ojph::param_cod cod = this->_codestream.access_cod (); + + cod.set_color_transform (this->_num_comps == 3 || this->_num_comps == 4); + cod.set_reversible (true); + + this->_output.open (); + + this->_codestream.write_headers (&this->_output); + + const char* line = inPtr; + const ojph::ui32 pixel_size = this->_num_comps * pixelTypeSize (HALF); + ojph::ui32 next_comp = 0; + ojph::line_buf* cur_line = this->_codestream.exchange (NULL, next_comp); + + assert (inSize == (pixel_size * height * width)); + + for (ojph::ui32 i = 0; i < height; ++i) + { + const char* in_buf; + + for (ojph::ui32 c = 0; c < this->_num_comps; c++) + { + assert (next_comp == c); + + in_buf = line + pixelTypeSize (HALF) * c; + + for (uint32_t p = 0; p < width; p++) + { + cur_line->i32[p] = + *reinterpret_cast (in_buf); + in_buf += pixel_size; + } + + cur_line = this->_codestream.exchange (cur_line, next_comp); + } + + line = in_buf; + } + + this->_codestream.flush (); + + assert (this->_output.tell () >= 0); + + outPtr = reinterpret_cast (this->_output.get_data ()); + + return static_cast (this->_output.tell ()); +} + +int +HTCompressor::compressTile ( + const char* inPtr, + int inSize, + IMATH_NAMESPACE::Box2i range, + const char*& outPtr) +{ + assert (0); +} + +int +HTCompressor::uncompress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) +{ + this->_input.open (reinterpret_cast (inPtr), inSize); + this->_codestream.read_headers (&this->_input); + + ojph::param_siz siz = this->_codestream.access_siz (); + ojph::ui32 width = siz.get_image_extent ().x - siz.get_image_offset ().x; + ojph::ui32 height = siz.get_image_extent ().y - siz.get_image_offset ().y; + + assert (this->_num_comps == siz.get_num_components ()); + + this->_codestream.set_planar (false); + + this->_codestream.create (); + + const ojph::ui32 pixel_size = this->_num_comps * pixelTypeSize (HALF); + + this->_buffer = new char[pixel_size * width * height]; + + char* line_buf = this->_buffer; + + for (uint32_t i = 0; i < height; ++i) + { + char* pixel_buf; + + for (uint32_t c = 0; c < this->_num_comps; c++) + { + ojph::ui32 next_comp = 0; + ojph::line_buf* cur_line = this->_codestream.pull (next_comp); + assert (next_comp == c); + + char* pixel_buf = line_buf + c * pixel_size; + ojph::si32* in = cur_line->i32; + + for (uint32_t p = 0; p < width; p++) + { + *reinterpret_cast (pixel_buf) = in[p]; + pixel_buf += pixel_size; + } + } + + line_buf = pixel_buf; + } + + outPtr = this->_buffer; + + return pixel_size * width * height; +} + +int +HTCompressor::uncompressTile ( + const char* inPtr, + int inSize, + IMATH_NAMESPACE::Box2i range, + const char*& outPtr) +{ + assert (0); +} + +OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT diff --git a/src/lib/OpenEXR/ImfHTCompressor.h b/src/lib/OpenEXR/ImfHTCompressor.h new file mode 100644 index 00000000..1cb27789 --- /dev/null +++ b/src/lib/OpenEXR/ImfHTCompressor.h @@ -0,0 +1,83 @@ +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + +#ifndef INCLUDED_IMF_HT_COMPRESSOR_H +#define INCLUDED_IMF_HT_COMPRESSOR_H + +//----------------------------------------------------------------------------- +// +// class HTCompressor -- uses High-Throughput JPEG 2000. +// +//----------------------------------------------------------------------------- + +#include "ImfNamespace.h" + +#include "ImfCompressor.h" + +#include +#include + +OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER + +class HTCompressor : public Compressor +{ +public: + HTCompressor (const Header& hdr); + + virtual ~HTCompressor (); + + HTCompressor (const HTCompressor& other) = delete; + HTCompressor& operator= (const HTCompressor& other) = delete; + HTCompressor (HTCompressor&& other) = delete; + HTCompressor& operator= (HTCompressor&& other) = delete; + + virtual int numScanLines () const; + + virtual Format format () const; + + virtual int + compress (const char* inPtr, int inSize, int minY, const char*& outPtr); + + virtual int compressTile ( + const char* inPtr, + int inSize, + IMATH_NAMESPACE::Box2i range, + const char*& outPtr); + + virtual int + uncompress (const char* inPtr, int inSize, int minY, const char*& outPtr); + + virtual int uncompressTile ( + const char* inPtr, + int inSize, + IMATH_NAMESPACE::Box2i range, + const char*& outPtr); + +private: + struct ChannelData; + + int compress ( + const char* inPtr, + int inSize, + IMATH_NAMESPACE::Box2i range, + const char*& outPtr); + + int uncompress ( + const char* inPtr, + int inSize, + IMATH_NAMESPACE::Box2i range, + const char*& outPtr); + + ojph::codestream _codestream; + ojph::mem_outfile _output; + ojph::mem_infile _input; + int _num_comps; + char* _buffer; + +}; + +OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT + +#endif diff --git a/src/test/OpenEXRTest/CMakeLists.txt b/src/test/OpenEXRTest/CMakeLists.txt index f91e6160..ea3cce7a 100644 --- a/src/test/OpenEXRTest/CMakeLists.txt +++ b/src/test/OpenEXRTest/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(OpenEXRTest testFutureProofing.h testHeader.cpp testHeader.h + testHTCompression.cpp testHuf.cpp testHuf.h testIDManifest.cpp @@ -172,6 +173,7 @@ define_openexr_tests( testExistingStreams testFutureProofing testHeader + testHTCompression testHuf testInputPart testIsComplete diff --git a/src/test/OpenEXRTest/main.cpp b/src/test/OpenEXRTest/main.cpp index 942c4b2d..971efc1d 100644 --- a/src/test/OpenEXRTest/main.cpp +++ b/src/test/OpenEXRTest/main.cpp @@ -33,6 +33,7 @@ #include "testExistingStreams.h" #include "testFutureProofing.h" #include "testHeader.h" +#include "testHTCompression.h" #include "testHuf.h" #include "testIDManifest.h" #include "testInputPart.h" @@ -190,6 +191,7 @@ main (int argc, char* argv[]) TEST (testCustomAttributes, "core"); TEST (testLineOrder, "basic"); TEST (testCompression, "basic"); + TEST (testHTCompression, "basic"); TEST (testCopyPixels, "basic"); TEST (testLut, "basic"); TEST (testSampleImages, "basic"); diff --git a/src/test/OpenEXRTest/testHTCompression.cpp b/src/test/OpenEXRTest/testHTCompression.cpp new file mode 100644 index 00000000..cbb60b72 --- /dev/null +++ b/src/test/OpenEXRTest/testHTCompression.cpp @@ -0,0 +1,366 @@ +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + +#ifdef NDEBUG +# undef NDEBUG +#endif + +#include "compareB44.h" + +#include "compareFloat.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace IMF = OPENEXR_IMF_NAMESPACE; +using namespace IMF; +using namespace std; +using namespace IMATH_NAMESPACE; + +namespace +{ + +struct pixelArray +{ + Array2D h; + Array2D rgba[4]; + pixelArray (int height, int width) : h (height, width) + { + for (int c = 0; c < 4; ++c) + { + rgba[c].resizeErase (height, width); + } + } +}; + +void +fillPixels1 (pixelArray& array, int width, int height) +{ + cout << "only zeroes" << endl; + + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) + { + array.h[y][x] = 0; + for (int c = 0; c < 4; ++c) + { + array.rgba[c][y][x] = 0; + } + } +} + +void +fillPixels2 (pixelArray& array, int width, int height) +{ + cout << "pattern 1" << endl; + + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) + { + array.h[y][x] = (x + y) & 1; + for (int c = 0; c < 4; ++c) + { + array.rgba[c][y][x] = (x + y) & 1; + } + } +} + +void +fillPixels3 (pixelArray& array, int width, int height) +{ + cout << "pattern 2" << endl; + + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) + { + + array.h[y][x] = sin (double (x)) + sin (y * 0.5); + for (int c = 0; c < 4; ++c) + { + array.rgba[c][y][x] = sin (double (x + c)) + sin (y * 0.5); + } + } +} + +void +fillPixels4 (pixelArray& array, int width, int height) +{ + cout << "random bits" << endl; + + // + // Use of a union to extract the bit pattern from a float, as is + // done below, works only if int and float have the same size. + // + + assert (sizeof (int) == sizeof (float)); + + Rand48 rand; + + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) + { + + array.h[y][x].setBits (rand.nexti ()); + for (int c = 0; c < 4; ++c) + { + array.rgba[c][y][x].setBits (rand.nexti ()); + } + + union + { + int i; + float f; + } u; + u.i = rand.nexti (); + } +} + +void +writeRead ( + pixelArray& array1, + const char fileName[], + int width, + int height, + Compression comp) +{ + + // + // Write the pixel data in pi1, ph1 and ph2 to an + // image file using the specified compression type + // and subsampling rates. Read the pixel data back + // from the file and verify that the data did not + // change. + // + + cout << "compression " << comp << flush; + + Header hdr ( + (Box2i ( + V2i (0, 0), // display window + V2i (width - 1, height - 1))), + (Box2i ( + V2i (0, 0), // data window + V2i (width - 1, height - 1)))); + + hdr.compression () = comp; + hdr.zipCompressionLevel () = 4; + + static const char* channels[] = {"R", "G", "B", "A", "H"}; + + for (int c = 0; c < 5; ++c) + { + hdr.channels ().insert ( + channels[c], // name + Channel ( + IMF::HALF, // type + 1, // xSampling + 1) // ySampling + ); + } + + { + FrameBuffer fb; + + fb.insert ( + "H", // name + Slice ( + IMF::HALF, // type + (char*) &array1.h[0][0], // base + sizeof (array1.h[0][0]), // xStride + sizeof (array1.h[0][0]) * width, // yStride + 1, // xSampling + 1) // ySampling + ); + + for (int c = 0; c < 4; c++) + { + fb.insert ( + channels[c], // name + Slice ( + IMF::HALF, // type + (char*) &array1.rgba[c][0][0], // base + sizeof (array1.rgba[c][0][0]), // xStride + sizeof (array1.rgba[c][0][0]) * width, // yStride + 1, // xSampling + 1) // ySampling + ); + } + + cout << " writing" << flush; + + remove (fileName); + + OutputFile out (fileName, hdr); + out.setFrameBuffer (fb); + out.writePixels (height); + } + + { + cout << " reading" << flush; + InputFile in (fileName); + + const Box2i& dw = hdr.dataWindow (); + int w = dw.max.x - dw.min.x + 1; + int h = dw.max.y - dw.min.y + 1; + int dx = dw.min.x; + int dy = dw.min.y; + + pixelArray array2 (h, w); + FrameBuffer fb; + + { + int xs = in.header ().channels ()["H"].xSampling; + int ys = in.header ().channels ()["H"].ySampling; + + fb.insert ( + "H", // name + Slice ( + IMF::HALF, // type + (char*) &array2.h[-dy][-dx], // base + sizeof (array2.h[0][0]), // xStride + sizeof (array2.h[0][0]) * w, // yStride + 1, // xSampling + 1) // ySampling + ); + } + + for (int c = 0; c < 4; ++c) + { + int xs = in.header ().channels ()[channels[c]].xSampling; + int ys = in.header ().channels ()[channels[c]].ySampling; + + fb.insert ( + channels[c], // name + Slice ( + IMF::HALF, // type + (char*) &array2.rgba[c][-dy][-dx], // base + sizeof (array2.rgba[c][0][0]), // xStride + sizeof (array2.rgba[c][0][0]) * (w), // yStride + 1, // xSampling + 1) // ySampling + ); + } + + in.setFrameBuffer (fb); + in.readPixels (dw.min.y, dw.max.y); + + cout << " comparing" << flush; + + assert (in.header ().displayWindow () == hdr.displayWindow ()); + assert (in.header ().dataWindow () == hdr.dataWindow ()); + assert (in.header ().pixelAspectRatio () == hdr.pixelAspectRatio ()); + assert ( + in.header ().screenWindowCenter () == hdr.screenWindowCenter ()); + assert (in.header ().screenWindowWidth () == hdr.screenWindowWidth ()); + assert (in.header ().lineOrder () == hdr.lineOrder ()); + assert (in.header ().compression () == hdr.compression ()); + + ChannelList::ConstIterator hi = hdr.channels ().begin (); + ChannelList::ConstIterator ii = in.header ().channels ().begin (); + + while (hi != hdr.channels ().end ()) + { + assert (!strcmp (hi.name (), ii.name ())); + assert (hi.channel ().type == ii.channel ().type); + assert (hi.channel ().xSampling == ii.channel ().xSampling); + assert (hi.channel ().ySampling == ii.channel ().ySampling); + + ++hi; + ++ii; + } + + assert (ii == in.header ().channels ().end ()); + + for (int y = 0; y < h; ++y) + { + for (int x = 0; x < w; ++x) + { + + if (!isLossyCompression (comp)) + { + assert (array1.h[y][x].bits () == array2.h[y][x].bits ()); + for (int c = 0; c < 4; ++c) + { + assert ( + array1.rgba[c][y][x].bits () == + array2.rgba[c][y][x].bits ()); + } + } + } + } + } + + remove (fileName); + cout << endl; +} + +void +writeRead ( + const std::string& tempDir, pixelArray& array, int w, int h, int dx, int dy) +{ + std::string filename = tempDir + "imf_test_comp.exr"; + + writeRead (array, filename.c_str (), w, h, HT_COMPRESSION); +} + +} // namespace + +void +testHTCompression (const std::string& tempDir) +{ + try + { + cout << "Testing pixel data types, " + "subsampling and " + "compression schemes" + << endl; + + const int W = 1371; + const int H = 159; + const int DX = 17; + const int DY = 29; + + pixelArray array (H, W); + + // + // If the following assertion fails, new pixel types have + // been added to the Imf library; testing code for the new + // pixel types should be added to this file. + // + + assert (NUM_PIXELTYPES == 3); + + fillPixels1 (array, W, H); + writeRead (tempDir, array, W, H, DX, DY); + + fillPixels2 (array, W, H); + writeRead (tempDir, array, W, H, DX, DY); + + fillPixels3 (array, W, H); + writeRead (tempDir, array, W, H, DX, DY); + + fillPixels4 (array, W, H); + writeRead (tempDir, array, W, H, DX, DY); + + cout << "ok\n" << endl; + } + catch (const std::exception& e) + { + cerr << "ERROR -- caught exception: " << e.what () << endl; + assert (false); + } +} diff --git a/src/test/OpenEXRTest/testHTCompression.h b/src/test/OpenEXRTest/testHTCompression.h new file mode 100644 index 00000000..1e4ec00a --- /dev/null +++ b/src/test/OpenEXRTest/testHTCompression.h @@ -0,0 +1,8 @@ +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + +#include + +void testHTCompression (const std::string& tempDir); From 85b395d481e4745c8f5db68c32cc8502cabee376 Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Thu, 15 Feb 2024 14:58:23 -0800 Subject: [PATCH 02/29] Fixing bugs --- src/lib/OpenEXR/ImfHTCompressor.cpp | 19 ++++++------------- src/lib/OpenEXR/ImfHTCompressor.h | 2 +- src/lib/OpenEXR/ImfScanLineInputFile.cpp | 2 +- src/test/OpenEXRTest/testHTCompression.cpp | 2 +- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/lib/OpenEXR/ImfHTCompressor.cpp b/src/lib/OpenEXR/ImfHTCompressor.cpp index d8f3aaa5..fc346217 100644 --- a/src/lib/OpenEXR/ImfHTCompressor.cpp +++ b/src/lib/OpenEXR/ImfHTCompressor.cpp @@ -163,38 +163,31 @@ HTCompressor::uncompress ( this->_codestream.create (); - const ojph::ui32 pixel_size = this->_num_comps * pixelTypeSize (HALF); + assert(sizeof(uint16_t) == pixelTypeSize (HALF)); - this->_buffer = new char[pixel_size * width * height]; + this->_buffer = new uint16_t[this->_num_comps * width * height]; - char* line_buf = this->_buffer; + uint16_t* line_buf = this->_buffer; for (uint32_t i = 0; i < height; ++i) { - char* pixel_buf; - for (uint32_t c = 0; c < this->_num_comps; c++) { ojph::ui32 next_comp = 0; ojph::line_buf* cur_line = this->_codestream.pull (next_comp); assert (next_comp == c); - char* pixel_buf = line_buf + c * pixel_size; - ojph::si32* in = cur_line->i32; - for (uint32_t p = 0; p < width; p++) { - *reinterpret_cast (pixel_buf) = in[p]; - pixel_buf += pixel_size; + this->_buffer[i * width * this->_num_comps + c + p] = (uint16_t) (cur_line->i32[p]); } } - line_buf = pixel_buf; } - outPtr = this->_buffer; + outPtr = (const char*) this->_buffer; - return pixel_size * width * height; + return this->_num_comps * pixelTypeSize (HALF) * width * height; } int diff --git a/src/lib/OpenEXR/ImfHTCompressor.h b/src/lib/OpenEXR/ImfHTCompressor.h index 1cb27789..bf46eb63 100644 --- a/src/lib/OpenEXR/ImfHTCompressor.h +++ b/src/lib/OpenEXR/ImfHTCompressor.h @@ -74,7 +74,7 @@ class HTCompressor : public Compressor ojph::mem_outfile _output; ojph::mem_infile _input; int _num_comps; - char* _buffer; + uint16_t* _buffer; }; diff --git a/src/lib/OpenEXR/ImfScanLineInputFile.cpp b/src/lib/OpenEXR/ImfScanLineInputFile.cpp index adaaa9f8..199d4f36 100644 --- a/src/lib/OpenEXR/ImfScanLineInputFile.cpp +++ b/src/lib/OpenEXR/ImfScanLineInputFile.cpp @@ -1149,7 +1149,7 @@ ScanLineInputFile::initialize (const Header& header) Compression comp = _data->header.compression (); - _data->linesInBuffer = numLinesInBuffer (comp); + _data->linesInBuffer = comp == HT_COMPRESSION ? (_data->maxY - _data->minY + 1) : numLinesInBuffer (comp); uint64_t lineOffsetSize = (static_cast(dataWindow.max.y) - static_cast(dataWindow.min.y) + static_cast(_data->linesInBuffer)) / diff --git a/src/test/OpenEXRTest/testHTCompression.cpp b/src/test/OpenEXRTest/testHTCompression.cpp index cbb60b72..4358c9e9 100644 --- a/src/test/OpenEXRTest/testHTCompression.cpp +++ b/src/test/OpenEXRTest/testHTCompression.cpp @@ -74,7 +74,7 @@ fillPixels2 (pixelArray& array, int width, int height) array.h[y][x] = (x + y) & 1; for (int c = 0; c < 4; ++c) { - array.rgba[c][y][x] = (x + y) & 1; + array.rgba[c][y][x] = array.h[y][x]; } } } From 23b5b39b032facf0346e77995cb070bf1d84d785 Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Thu, 15 Feb 2024 16:48:05 -0800 Subject: [PATCH 03/29] Works --- src/lib/OpenEXR/ImfHTCompressor.cpp | 17 ++++------- src/test/OpenEXRTest/testHTCompression.cpp | 34 +++++++++++----------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/lib/OpenEXR/ImfHTCompressor.cpp b/src/lib/OpenEXR/ImfHTCompressor.cpp index fc346217..ddeab42e 100644 --- a/src/lib/OpenEXR/ImfHTCompressor.cpp +++ b/src/lib/OpenEXR/ImfHTCompressor.cpp @@ -97,13 +97,12 @@ HTCompressor::compress ( this->_codestream.write_headers (&this->_output); - const char* line = inPtr; - const ojph::ui32 pixel_size = this->_num_comps * pixelTypeSize (HALF); + assert (inSize == ( this->_num_comps * pixelTypeSize (HALF) * height * width)); + + const uint16_t* pixels = (const uint16_t*) inPtr; ojph::ui32 next_comp = 0; ojph::line_buf* cur_line = this->_codestream.exchange (NULL, next_comp); - assert (inSize == (pixel_size * height * width)); - for (ojph::ui32 i = 0; i < height; ++i) { const char* in_buf; @@ -112,19 +111,15 @@ HTCompressor::compress ( { assert (next_comp == c); - in_buf = line + pixelTypeSize (HALF) * c; - for (uint32_t p = 0; p < width; p++) { cur_line->i32[p] = - *reinterpret_cast (in_buf); - in_buf += pixel_size; + static_cast (*pixels++); } cur_line = this->_codestream.exchange (cur_line, next_comp); } - line = in_buf; } this->_codestream.flush (); @@ -167,7 +162,7 @@ HTCompressor::uncompress ( this->_buffer = new uint16_t[this->_num_comps * width * height]; - uint16_t* line_buf = this->_buffer; + uint16_t* pixel = this->_buffer; for (uint32_t i = 0; i < height; ++i) { @@ -179,7 +174,7 @@ HTCompressor::uncompress ( for (uint32_t p = 0; p < width; p++) { - this->_buffer[i * width * this->_num_comps + c + p] = (uint16_t) (cur_line->i32[p]); + *pixel++ = (uint16_t) (cur_line->i32[p]); } } diff --git a/src/test/OpenEXRTest/testHTCompression.cpp b/src/test/OpenEXRTest/testHTCompression.cpp index 4358c9e9..50e06551 100644 --- a/src/test/OpenEXRTest/testHTCompression.cpp +++ b/src/test/OpenEXRTest/testHTCompression.cpp @@ -131,7 +131,7 @@ fillPixels4 (pixelArray& array, int width, int height) void writeRead ( - pixelArray& array1, + pixelArray& ref_array, const char fileName[], int width, int height, @@ -179,9 +179,9 @@ writeRead ( "H", // name Slice ( IMF::HALF, // type - (char*) &array1.h[0][0], // base - sizeof (array1.h[0][0]), // xStride - sizeof (array1.h[0][0]) * width, // yStride + (char*) &ref_array.h[0][0], // base + sizeof (ref_array.h[0][0]), // xStride + sizeof (ref_array.h[0][0]) * width, // yStride 1, // xSampling 1) // ySampling ); @@ -192,9 +192,9 @@ writeRead ( channels[c], // name Slice ( IMF::HALF, // type - (char*) &array1.rgba[c][0][0], // base - sizeof (array1.rgba[c][0][0]), // xStride - sizeof (array1.rgba[c][0][0]) * width, // yStride + (char*) &ref_array.rgba[c][0][0], // base + sizeof (ref_array.rgba[c][0][0]), // xStride + sizeof (ref_array.rgba[c][0][0]) * width, // yStride 1, // xSampling 1) // ySampling ); @@ -219,7 +219,7 @@ writeRead ( int dx = dw.min.x; int dy = dw.min.y; - pixelArray array2 (h, w); + pixelArray decoded_array (h, w); FrameBuffer fb; { @@ -230,9 +230,9 @@ writeRead ( "H", // name Slice ( IMF::HALF, // type - (char*) &array2.h[-dy][-dx], // base - sizeof (array2.h[0][0]), // xStride - sizeof (array2.h[0][0]) * w, // yStride + (char*) &decoded_array.h[-dy][-dx], // base + sizeof (decoded_array.h[0][0]), // xStride + sizeof (decoded_array.h[0][0]) * w, // yStride 1, // xSampling 1) // ySampling ); @@ -247,9 +247,9 @@ writeRead ( channels[c], // name Slice ( IMF::HALF, // type - (char*) &array2.rgba[c][-dy][-dx], // base - sizeof (array2.rgba[c][0][0]), // xStride - sizeof (array2.rgba[c][0][0]) * (w), // yStride + (char*) &decoded_array.rgba[c][-dy][-dx], // base + sizeof (decoded_array.rgba[c][0][0]), // xStride + sizeof (decoded_array.rgba[c][0][0]) * (w), // yStride 1, // xSampling 1) // ySampling ); @@ -292,12 +292,12 @@ writeRead ( if (!isLossyCompression (comp)) { - assert (array1.h[y][x].bits () == array2.h[y][x].bits ()); + assert (ref_array.h[y][x].bits () == decoded_array.h[y][x].bits ()); for (int c = 0; c < 4; ++c) { assert ( - array1.rgba[c][y][x].bits () == - array2.rgba[c][y][x].bits ()); + ref_array.rgba[c][y][x].bits () == + decoded_array.rgba[c][y][x].bits ()); } } } From e22993a5f89b6f9bc4b64d27b276c4b3e0d3fb19 Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Fri, 16 Feb 2024 22:38:03 -0800 Subject: [PATCH 04/29] Added conversion tool Set codeblock size --- src/bin/CMakeLists.txt | 1 + src/bin/exrconv/CMakeLists.txt | 15 + src/bin/exrconv/main.cpp | 413 ++++++++++++++++++++++++++++ src/lib/OpenEXR/ImfHTCompressor.cpp | 2 + 4 files changed, 431 insertions(+) create mode 100644 src/bin/exrconv/CMakeLists.txt create mode 100644 src/bin/exrconv/main.cpp diff --git a/src/bin/CMakeLists.txt b/src/bin/CMakeLists.txt index 893fd58d..291ce126 100644 --- a/src/bin/CMakeLists.txt +++ b/src/bin/CMakeLists.txt @@ -16,4 +16,5 @@ if(OPENEXR_BUILD_TOOLS) add_subdirectory( exrmultipart ) add_subdirectory( exrcheck ) add_subdirectory( exrmanifest ) + add_subdirectory( exrconv ) endif() diff --git a/src/bin/exrconv/CMakeLists.txt b/src/bin/exrconv/CMakeLists.txt new file mode 100644 index 00000000..992f90a2 --- /dev/null +++ b/src/bin/exrconv/CMakeLists.txt @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenEXR Project. + +add_executable(exrconv main.cpp) +target_link_libraries(exrconv OpenEXR::OpenEXR) +set_target_properties(exrconv PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" +) + +if(OPENEXR_INSTALL_TOOLS) + install(TARGETS exrconv DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() +if(WIN32 AND (BUILD_SHARED_LIBS OR OPENEXR_BUILD_BOTH_STATIC_SHARED)) + target_compile_definitions(exrconv PRIVATE OPENEXR_DLL) +endif() diff --git a/src/bin/exrconv/main.cpp b/src/bin/exrconv/main.cpp new file mode 100644 index 00000000..e104afb9 --- /dev/null +++ b/src/bin/exrconv/main.cpp @@ -0,0 +1,413 @@ +#include +#include +#include + +#include "ImfArray.h" +#include "ImfCompression.h" +#include "ImfHeader.h" +#include "ImfRgbaFile.h" +#include "ImfFrameBuffer.h" +#include +#include + +namespace IMF = OPENEXR_IMF_NAMESPACE; +using namespace OPENEXR_IMF_NAMESPACE; +using namespace IMATH_NAMESPACE; + +typedef struct _commandline_args +{ + char input_filename[512]; + char output_filename[512]; + unsigned int start_frame; + unsigned int end_frame; + char compression_string[512]; + bool is_process_framerange; +} commandline_args_t; + +static const char *compression_string_table[NUM_COMPRESSION_METHODS] = { +"NO_COMPRESSION", // = 0 // no compression +"RLE_COMPRESSION", // = 1 // run length encoding +"ZIPS_COMPRESSION",// = 2 // zlib compression, one scan line at a time +"ZIP_COMPRESSION", // = 3, // zlib compression, in blocks of 16 scan lines +"PIZ_COMPRESSION", // = 4, // piz-based wavelet compression +"PXR24_COMPRESSION", // = 5, // lossy 24-bit float compression +"B44_COMPRESSION", // = 6, // lossy 4-by-4 pixel block compression, + // fixed compression rate +"B44A_COMPRESSION", // = 7, // lossy 4-by-4 pixel block compression, + // flat fields are compressed more +"DWAA_COMPRESSION", // = 8, // lossy DCT based compression, in blocks + // of 32 scanlines. More efficient for partial + // buffer access. +"DWAB_COMPRESSION", // = 9, // lossy DCT based compression, in blocks + // of 256 scanlines. More efficient space + // wise and faster to decode full frames + // than DWAA_COMPRESSION. +"HT_COMPRESSION" +}; + +void print_argument_list(int argc, char* argv[]) +{ + fprintf(stderr, "argument list:\n"); + for (int i = 0; i < argc; i++) + fprintf(stderr, "\t argv[%d] = %s\n", i, argv[i]); + return; +} + + + +void print_allowed_compression_strings() +{ + for (int i = 0; i < (int)NUM_COMPRESSION_METHODS; i++) + { + fprintf(stderr, "\t %s\n", compression_string_table[i]); + } + + return; +} + +void print_usage(int argc, char* argv[]) +{ + fprintf(stderr, "This program converts exr images to exr images with different image compression\n"); + fprintf(stderr, "USAGE: %s \n", argv[0]); + fprintf(stderr, "\nREQUIRED ARGUMENTS:\n"); + fprintf(stderr, " -i - exr filename - use printf() style formatting for an input sequence, e.g. input.%%06d.exr\n"); + fprintf(stderr, " -o - exr filename - use printf() style formatting for an output sequence e.g. output.%%06d.exr\n"); + fprintf(stderr, "\nOPTIONAL ARGUMENTS:\n"); + fprintf(stderr, " -c - use this to specify different image compression for the output\n"); + print_allowed_compression_strings(); + fprintf(stderr, " -s \n"); + fprintf(stderr, " -e \n"); + + fprintf(stderr, "\nUSAGE EXAMPLE: %s -i input.exr -o output.%%01d.exr\n", argv[0]); + fprintf(stderr, "\nUSAGE EXAMPLE: %s -i input.%%06d.exr -o output.%%07d.exr -s 3 -e 450 -c ZIP_COMPRESSION\n", argv[0]); + + + if (argc != 1) + { + fprintf(stderr, "\n"); + print_argument_list(argc, argv); + } + + fprintf(stderr, "\nVERSION INFO: This software executable was compiled on %s at %s\n", __DATE__, __TIME__); + + return; +} + +int process_commandline_args(int argc, char* argv[], commandline_args_t *args) +{ + // set defaults + memset(args, 0, sizeof(commandline_args_t)); + + bool is_i_parsed = false; + bool is_o_parsed = false; + bool is_c_parsed = false; + bool is_s_parsed = false; + bool is_e_parsed = false; + + if (argc == 1) + { + print_usage(argc, argv); + return -1; + } + + for (int i = 1; i < argc; i++) + { + int args_remaining = argc - i; + + // usage + if (strcmp(argv[i], "-u") == 0) + { + print_usage(argc, argv); + exit(1); + } + + // input filename + else if (strcmp(argv[i], "-i") == 0) + { + if (args_remaining <= 1) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -i missing filename argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else if (true == is_i_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -i argument has already been processed, -i should only be used once\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else + { + strcpy(args->input_filename, argv[i + 1]); + i = i + 1; + is_i_parsed = true; + } + } + + // output filename + else if (strcmp(argv[i], "-o") == 0) + { + if (args_remaining <= 1) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -o missing filename argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else if (true == is_o_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -o argument has already been processed, -o should only be used once\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else + { + strcpy(args->output_filename, argv[i + 1]); + i = i + 1; + is_o_parsed = true; + } + } + + // start frame + else if (strcmp(argv[i], "-s") == 0) + { + if (args_remaining <= 1) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -s missing start frame argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else if (true == is_s_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -s argument has already been processed, -s should only be used once\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else + { + args->start_frame = (unsigned int)atoi(argv[i + 1]); + i = i + 1; + is_s_parsed = true; + } + } + + // end frame + else if (strcmp(argv[i], "-e") == 0) + { + if (args_remaining <= 1) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -e missing end frame argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else if (true == is_e_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -e argument has already been processed, -e should only be used once\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else + { + args->end_frame = (unsigned int)atoi(argv[i + 1]); + i = i + 1; + is_e_parsed = true; + } + } + + // compression + else if (strcmp(argv[i], "-c") == 0) + { + if (args_remaining <= 1) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -c missing compression string argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else if (true == is_c_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -c argument has already been processed, -c should only be used once\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + else + { + strcpy(args->compression_string, argv[i + 1]); + i = i + 1; + is_c_parsed = true; + } + } + + // unrecognized arguments + else + { + fprintf(stderr, "COMMANDLINE PROCESSING ERROR: argv[%d] = %s, this is an unrecognized command-line argument\n", i, argv[i]); + exit(1); + } + } // end of argv[] processing loop + + + // check that input filename and output filename are processed + if (true != is_i_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -o is missing but is a required argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + if (true != is_o_parsed) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n -o is missing but is a required argument\n", __FUNCTION__, __FILE__, __LINE__); + return -1; + } + + // if one of end-frame/start-frame arguments is processed, then make sure all of them are processed + if ((true == is_s_parsed) || (true == is_e_parsed)) + { + args->is_process_framerange = true; + + if ((true != is_s_parsed) || (true != is_e_parsed)) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n", __FUNCTION__, __FILE__, __LINE__); + fprintf(stderr, " if one of the following arguments are used, \n -s -e, then all the arguments must be used.\n"); + + if (true == is_s_parsed) + fprintf(stderr, "-sf1 was used\n"); + else + fprintf(stderr, "-sf1 was not used\n"); + + if (true == is_e_parsed) + fprintf(stderr, "-ef1 was used\n"); + else + fprintf(stderr, "-ef1 was not used\n"); + + return -1; + } + } + + return 0; +} + +int check_commandline_args(commandline_args_t *args) +{ + + if (true == args->is_process_framerange) + { + if (args->end_frame < args->start_frame) + { + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n", __FUNCTION__, __FILE__, __LINE__); + fprintf(stderr, " args->end_frame should not be less than args->start_frame\n"); + fprintf(stderr, " args->start_frame = %d args->end_frame = %d\n", args->start_frame, args->end_frame); + return -1; + } + } + + // check that the we have access to the range of input files specified + const unsigned int number_of_frames_to_process = args->end_frame - args->start_frame + 1; + + // check the specified input frames + for (unsigned int frame_index = 0; frame_index < number_of_frames_to_process; frame_index++) + { + if (0 == frame_index) + { + fprintf(stderr, "Start checking %d input files . . .\n", number_of_frames_to_process); + } + + // make image filename + char input_filename[512] = { '\0' }; + sprintf(input_filename, args->input_filename, frame_index + args->start_frame); + + // try to open image file + { + FILE *input_file = NULL; + input_file = fopen(input_filename, "rb"); + if (NULL == input_file) + { + fprintf(stderr, "ERROR in file on line %d of %s in function %s(): unable to open input filename = %s for binary reading\n", + __LINE__, __FILE__, __FUNCTION__, input_filename); + return false; + } + else + { + // we can open the image file OK + fclose(input_file); + } + } + + } + fprintf(stderr, "Finished checking %d input files\n", number_of_frames_to_process); + + if ( '\0' == args->compression_string[0]) + { + strcpy(args->compression_string, compression_string_table[0]); + fprintf(stderr, "USAGE WARNING in function %s of file %s on line %d:\n", __FUNCTION__, __FILE__, __LINE__); + fprintf(stderr, " -c argument not provided, using c = %s compression value.\n", args->compression_string); + } + + return 0; +} + +Compression get_compression_from_compression_string(const char* compression_string) +{ + for (int i = 0; i < (int)NUM_COMPRESSION_METHODS; i++) + { + if (0 == strcmp(compression_string, compression_string_table[i])) + { + return (Compression) i; + } + } + + fprintf(stderr, "USAGE ERROR in function %s of file %s on line %d:\n", __FUNCTION__, __FILE__, __LINE__); + fprintf(stderr, " -c %s is not on the list of supported compression strings. \nThe supported list of compression values is shown below:\n", compression_string); + print_allowed_compression_strings(); + fprintf(stderr, "Exiting.\n"); + exit(-1); + + return NO_COMPRESSION; +} + +int main(int argc, char* argv[]) +{ + // process command line arguments + commandline_args_t args; + int process_command_args_ok = process_commandline_args(argc, argv, &args); + if (process_command_args_ok != 0) + { + fprintf(stderr, "\n"); + print_argument_list(argc, argv); + fprintf(stderr, "\n"); + fprintf(stderr, "use -u argument to print usage info\n"); + exit(-1); + } + + // check command arguments + int check_command_args_ok = check_commandline_args(&args); + if (check_command_args_ok != 0) + { + fprintf(stderr, "USAGE ERROR: problem with checking command-line args\n"); + fprintf(stderr, "\n"); + print_argument_list(argc, argv); + fprintf(stderr, "use -u argument to print usage info\n"); + exit(1); + } + + Compression selected_compression = get_compression_from_compression_string(args.compression_string); + + unsigned int number_of_frames_to_process = args.end_frame - args.start_frame + 1; + for (unsigned int frame_index = 0; frame_index < number_of_frames_to_process; frame_index++) + { + // make image filenames + char input_filename[512] = { '\0' }; + char output_filename[512] = { '\0' }; + sprintf(input_filename, args.input_filename, frame_index + args.start_frame); + sprintf(output_filename, args.output_filename, frame_index + args.start_frame); + + RgbaInputFile i_file(input_filename); + + Box2i dw = i_file.dataWindow (); + int width = dw.max.x - dw.min.x + 1; + int height = dw.max.y - dw.min.y + 1; + + Array2D pixels (height, width); + + i_file.setFrameBuffer (&pixels[-dw.min.x][-dw.min.y], 1, width); + i_file.readPixels (dw.min.y, dw.max.y); + + Header header = i_file.header(); + header.compression () = selected_compression; + + + RgbaOutputFile o_file(output_filename, header); + o_file.setFrameBuffer(&pixels[-dw.min.x][-dw.min.y], 1, width); + o_file.writePixels(height); + + } + + return 0; +} \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfHTCompressor.cpp b/src/lib/OpenEXR/ImfHTCompressor.cpp index ddeab42e..b2629c4c 100644 --- a/src/lib/OpenEXR/ImfHTCompressor.cpp +++ b/src/lib/OpenEXR/ImfHTCompressor.cpp @@ -92,6 +92,8 @@ HTCompressor::compress ( cod.set_color_transform (this->_num_comps == 3 || this->_num_comps == 4); cod.set_reversible (true); + cod.set_block_dims(128, 32); + cod.set_num_decomposition(5); this->_output.open (); From f56477632b14a2d49d9bcdc859fdbb669b80b3b9 Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Fri, 16 Feb 2024 23:25:36 -0800 Subject: [PATCH 05/29] NLT --- src/bin/exrconv/main.cpp | 2 +- src/lib/OpenEXR/ImfHTCompressor.cpp | 19 +++++++++++-------- src/lib/OpenEXR/ImfHTCompressor.h | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/bin/exrconv/main.cpp b/src/bin/exrconv/main.cpp index e104afb9..a911d7a1 100644 --- a/src/bin/exrconv/main.cpp +++ b/src/bin/exrconv/main.cpp @@ -403,7 +403,7 @@ int main(int argc, char* argv[]) header.compression () = selected_compression; - RgbaOutputFile o_file(output_filename, header); + RgbaOutputFile o_file(output_filename, header, WRITE_RGB); o_file.setFrameBuffer(&pixels[-dw.min.x][-dw.min.y], 1, width); o_file.writePixels(height); diff --git a/src/lib/OpenEXR/ImfHTCompressor.cpp b/src/lib/OpenEXR/ImfHTCompressor.cpp index b2629c4c..8e78af0a 100644 --- a/src/lib/OpenEXR/ImfHTCompressor.cpp +++ b/src/lib/OpenEXR/ImfHTCompressor.cpp @@ -80,7 +80,7 @@ HTCompressor::compress ( siz.set_num_components (this->_num_comps); for (ojph::ui32 c = 0; c < this->_num_comps; c++) - siz.set_component (c, ojph::point (1, 1), 16, false); + siz.set_component (c, ojph::point (1, 1), 16, true); /* TODO: extend to multiple tiles and non-coincident data and display windows */ siz.set_image_offset (ojph::point (0, 0)); @@ -101,7 +101,7 @@ HTCompressor::compress ( assert (inSize == ( this->_num_comps * pixelTypeSize (HALF) * height * width)); - const uint16_t* pixels = (const uint16_t*) inPtr; + const int16_t* pixels = (const int16_t*) inPtr; ojph::ui32 next_comp = 0; ojph::line_buf* cur_line = this->_codestream.exchange (NULL, next_comp); @@ -115,8 +115,9 @@ HTCompressor::compress ( for (uint32_t p = 0; p < width; p++) { - cur_line->i32[p] = - static_cast (*pixels++); + ojph::si32 c = static_cast (*pixels++); + + cur_line->i32[p] = c < 0 ? -32769 - c : c; } cur_line = this->_codestream.exchange (cur_line, next_comp); @@ -160,11 +161,11 @@ HTCompressor::uncompress ( this->_codestream.create (); - assert(sizeof(uint16_t) == pixelTypeSize (HALF)); + assert(sizeof(int16_t) == pixelTypeSize (HALF)); - this->_buffer = new uint16_t[this->_num_comps * width * height]; + this->_buffer = new int16_t[this->_num_comps * width * height]; - uint16_t* pixel = this->_buffer; + int16_t* pixel = this->_buffer; for (uint32_t i = 0; i < height; ++i) { @@ -176,7 +177,9 @@ HTCompressor::uncompress ( for (uint32_t p = 0; p < width; p++) { - *pixel++ = (uint16_t) (cur_line->i32[p]); + ojph::si32 c = cur_line->i32[p]; + + *pixel++ = (int16_t) (c < 0 ? -32769 - c : c); } } diff --git a/src/lib/OpenEXR/ImfHTCompressor.h b/src/lib/OpenEXR/ImfHTCompressor.h index bf46eb63..4ef1775a 100644 --- a/src/lib/OpenEXR/ImfHTCompressor.h +++ b/src/lib/OpenEXR/ImfHTCompressor.h @@ -74,7 +74,7 @@ class HTCompressor : public Compressor ojph::mem_outfile _output; ojph::mem_infile _input; int _num_comps; - uint16_t* _buffer; + int16_t* _buffer; }; From 3d67dfaad526d03e3e1a23fb2e374da3273819e9 Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Sat, 17 Feb 2024 12:01:35 -0800 Subject: [PATCH 06/29] Add perf cli --- src/bin/CMakeLists.txt | 1 + src/bin/exrperf/CMakeLists.txt | 15 + src/bin/exrperf/cxxopts.hpp | 2714 +++++++++++++++++++++ src/bin/exrperf/main.cpp | 165 ++ src/lib/OpenEXR/ImfMultiPartInputFile.cpp | 1 + 5 files changed, 2896 insertions(+) create mode 100644 src/bin/exrperf/CMakeLists.txt create mode 100644 src/bin/exrperf/cxxopts.hpp create mode 100644 src/bin/exrperf/main.cpp diff --git a/src/bin/CMakeLists.txt b/src/bin/CMakeLists.txt index 291ce126..5c03f4e5 100644 --- a/src/bin/CMakeLists.txt +++ b/src/bin/CMakeLists.txt @@ -17,4 +17,5 @@ if(OPENEXR_BUILD_TOOLS) add_subdirectory( exrcheck ) add_subdirectory( exrmanifest ) add_subdirectory( exrconv ) + add_subdirectory( exrperf ) endif() diff --git a/src/bin/exrperf/CMakeLists.txt b/src/bin/exrperf/CMakeLists.txt new file mode 100644 index 00000000..317daa3e --- /dev/null +++ b/src/bin/exrperf/CMakeLists.txt @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenEXR Project. + +add_executable(exrperf main.cpp) +target_link_libraries(exrperf OpenEXR::OpenEXR) +set_target_properties(exrperf PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" +) + +if(OPENEXR_INSTALL_TOOLS) + install(TARGETS exrperf DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() +if(WIN32 AND (BUILD_SHARED_LIBS OR OPENEXR_BUILD_BOTH_STATIC_SHARED)) + target_compile_definitions(exrperf PRIVATE OPENEXR_DLL) +endif() diff --git a/src/bin/exrperf/cxxopts.hpp b/src/bin/exrperf/cxxopts.hpp new file mode 100644 index 00000000..2dd4da6b --- /dev/null +++ b/src/bin/exrperf/cxxopts.hpp @@ -0,0 +1,2714 @@ +/* + +Copyright (c) 2014, 2015, 2016, 2017 Jarryd Beck + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +#ifndef CXXOPTS_HPP_INCLUDED +#define CXXOPTS_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GNUC__) && !defined(__clang__) +# if (__GNUC__ * 10 + __GNUC_MINOR__) < 49 +# define CXXOPTS_NO_REGEX true +# endif +#endif + +#ifndef CXXOPTS_NO_REGEX +# include +#endif // CXXOPTS_NO_REGEX + +// Nonstandard before C++17, which is coincidentally what we also need for +#ifdef __has_include +# if __has_include() +# include +# ifdef __cpp_lib_optional +# define CXXOPTS_HAS_OPTIONAL +# endif +# endif +#endif + +#if __cplusplus >= 201603L +#define CXXOPTS_NODISCARD [[nodiscard]] +#else +#define CXXOPTS_NODISCARD +#endif + +#ifndef CXXOPTS_VECTOR_DELIMITER +#define CXXOPTS_VECTOR_DELIMITER ',' +#endif + +#define CXXOPTS__VERSION_MAJOR 3 +#define CXXOPTS__VERSION_MINOR 0 +#define CXXOPTS__VERSION_PATCH 0 + +#if (__GNUC__ < 10 || (__GNUC__ == 10 && __GNUC_MINOR__ < 1)) && __GNUC__ >= 6 + #define CXXOPTS_NULL_DEREF_IGNORE +#endif + +namespace cxxopts +{ + static constexpr struct { + uint8_t major, minor, patch; + } version = { + CXXOPTS__VERSION_MAJOR, + CXXOPTS__VERSION_MINOR, + CXXOPTS__VERSION_PATCH + }; +} // namespace cxxopts + +//when we ask cxxopts to use Unicode, help strings are processed using ICU, +//which results in the correct lengths being computed for strings when they +//are formatted for the help output +//it is necessary to make sure that can be found by the +//compiler, and that icu-uc is linked in to the binary. + +#ifdef CXXOPTS_USE_UNICODE +#include + +namespace cxxopts +{ + using String = icu::UnicodeString; + + inline + String + toLocalString(std::string s) + { + return icu::UnicodeString::fromUTF8(std::move(s)); + } + +#if defined(__GNUC__) +// GNU GCC with -Weffc++ will issue a warning regarding the upcoming class, we want to silence it: +// warning: base class 'class std::enable_shared_from_this' has accessible non-virtual destructor +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#pragma GCC diagnostic ignored "-Weffc++" +// This will be ignored under other compilers like LLVM clang. +#endif + class UnicodeStringIterator : public + std::iterator + { + public: + + UnicodeStringIterator(const icu::UnicodeString* string, int32_t pos) + : s(string) + , i(pos) + { + } + + value_type + operator*() const + { + return s->char32At(i); + } + + bool + operator==(const UnicodeStringIterator& rhs) const + { + return s == rhs.s && i == rhs.i; + } + + bool + operator!=(const UnicodeStringIterator& rhs) const + { + return !(*this == rhs); + } + + UnicodeStringIterator& + operator++() + { + ++i; + return *this; + } + + UnicodeStringIterator + operator+(int32_t v) + { + return UnicodeStringIterator(s, i + v); + } + + private: + const icu::UnicodeString* s; + int32_t i; + }; +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + + inline + String& + stringAppend(String&s, String a) + { + return s.append(std::move(a)); + } + + inline + String& + stringAppend(String& s, size_t n, UChar32 c) + { + for (size_t i = 0; i != n; ++i) + { + s.append(c); + } + + return s; + } + + template + String& + stringAppend(String& s, Iterator begin, Iterator end) + { + while (begin != end) + { + s.append(*begin); + ++begin; + } + + return s; + } + + inline + size_t + stringLength(const String& s) + { + return s.length(); + } + + inline + std::string + toUTF8String(const String& s) + { + std::string result; + s.toUTF8String(result); + + return result; + } + + inline + bool + empty(const String& s) + { + return s.isEmpty(); + } +} + +namespace std +{ + inline + cxxopts::UnicodeStringIterator + begin(const icu::UnicodeString& s) + { + return cxxopts::UnicodeStringIterator(&s, 0); + } + + inline + cxxopts::UnicodeStringIterator + end(const icu::UnicodeString& s) + { + return cxxopts::UnicodeStringIterator(&s, s.length()); + } +} + +//ifdef CXXOPTS_USE_UNICODE +#else + +namespace cxxopts +{ + using String = std::string; + + template + T + toLocalString(T&& t) + { + return std::forward(t); + } + + inline + size_t + stringLength(const String& s) + { + return s.length(); + } + + inline + String& + stringAppend(String&s, const String& a) + { + return s.append(a); + } + + inline + String& + stringAppend(String& s, size_t n, char c) + { + return s.append(n, c); + } + + template + String& + stringAppend(String& s, Iterator begin, Iterator end) + { + return s.append(begin, end); + } + + template + std::string + toUTF8String(T&& t) + { + return std::forward(t); + } + + inline + bool + empty(const std::string& s) + { + return s.empty(); + } +} // namespace cxxopts + +//ifdef CXXOPTS_USE_UNICODE +#endif + +namespace cxxopts +{ + namespace + { +#ifdef _WIN32 + const std::string LQUOTE("\'"); + const std::string RQUOTE("\'"); +#else + const std::string LQUOTE("‘"); + const std::string RQUOTE("’"); +#endif + } // namespace + +#if defined(__GNUC__) +// GNU GCC with -Weffc++ will issue a warning regarding the upcoming class, we want to silence it: +// warning: base class 'class std::enable_shared_from_this' has accessible non-virtual destructor +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#pragma GCC diagnostic ignored "-Weffc++" +// This will be ignored under other compilers like LLVM clang. +#endif + class Value : public std::enable_shared_from_this + { + public: + + virtual ~Value() = default; + + virtual + std::shared_ptr + clone() const = 0; + + virtual void + parse(const std::string& text) const = 0; + + virtual void + parse() const = 0; + + virtual bool + has_default() const = 0; + + virtual bool + is_container() const = 0; + + virtual bool + has_implicit() const = 0; + + virtual std::string + get_default_value() const = 0; + + virtual std::string + get_implicit_value() const = 0; + + virtual std::shared_ptr + default_value(const std::string& value) = 0; + + virtual std::shared_ptr + implicit_value(const std::string& value) = 0; + + virtual std::shared_ptr + no_implicit_value() = 0; + + virtual bool + is_boolean() const = 0; + }; +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + class OptionException : public std::exception + { + public: + explicit OptionException(std::string message) + : m_message(std::move(message)) + { + } + + CXXOPTS_NODISCARD + const char* + what() const noexcept override + { + return m_message.c_str(); + } + + private: + std::string m_message; + }; + + class OptionSpecException : public OptionException + { + public: + + explicit OptionSpecException(const std::string& message) + : OptionException(message) + { + } + }; + + class OptionParseException : public OptionException + { + public: + explicit OptionParseException(const std::string& message) + : OptionException(message) + { + } + }; + + class option_exists_error : public OptionSpecException + { + public: + explicit option_exists_error(const std::string& option) + : OptionSpecException("Option " + LQUOTE + option + RQUOTE + " already exists") + { + } + }; + + class invalid_option_format_error : public OptionSpecException + { + public: + explicit invalid_option_format_error(const std::string& format) + : OptionSpecException("Invalid option format " + LQUOTE + format + RQUOTE) + { + } + }; + + class option_syntax_exception : public OptionParseException { + public: + explicit option_syntax_exception(const std::string& text) + : OptionParseException("Argument " + LQUOTE + text + RQUOTE + + " starts with a - but has incorrect syntax") + { + } + }; + + class option_not_exists_exception : public OptionParseException + { + public: + explicit option_not_exists_exception(const std::string& option) + : OptionParseException("Option " + LQUOTE + option + RQUOTE + " does not exist") + { + } + }; + + class missing_argument_exception : public OptionParseException + { + public: + explicit missing_argument_exception(const std::string& option) + : OptionParseException( + "Option " + LQUOTE + option + RQUOTE + " is missing an argument" + ) + { + } + }; + + class option_requires_argument_exception : public OptionParseException + { + public: + explicit option_requires_argument_exception(const std::string& option) + : OptionParseException( + "Option " + LQUOTE + option + RQUOTE + " requires an argument" + ) + { + } + }; + + class option_not_has_argument_exception : public OptionParseException + { + public: + option_not_has_argument_exception + ( + const std::string& option, + const std::string& arg + ) + : OptionParseException( + "Option " + LQUOTE + option + RQUOTE + + " does not take an argument, but argument " + + LQUOTE + arg + RQUOTE + " given" + ) + { + } + }; + + class option_not_present_exception : public OptionParseException + { + public: + explicit option_not_present_exception(const std::string& option) + : OptionParseException("Option " + LQUOTE + option + RQUOTE + " not present") + { + } + }; + + class option_has_no_value_exception : public OptionException + { + public: + explicit option_has_no_value_exception(const std::string& option) + : OptionException( + !option.empty() ? + ("Option " + LQUOTE + option + RQUOTE + " has no value") : + "Option has no value") + { + } + }; + + class argument_incorrect_type : public OptionParseException + { + public: + explicit argument_incorrect_type + ( + const std::string& arg + ) + : OptionParseException( + "Argument " + LQUOTE + arg + RQUOTE + " failed to parse" + ) + { + } + }; + + class option_required_exception : public OptionParseException + { + public: + explicit option_required_exception(const std::string& option) + : OptionParseException( + "Option " + LQUOTE + option + RQUOTE + " is required but not present" + ) + { + } + }; + + template + void throw_or_mimic(const std::string& text) + { + static_assert(std::is_base_of::value, + "throw_or_mimic only works on std::exception and " + "deriving classes"); + +#ifndef CXXOPTS_NO_EXCEPTIONS + // If CXXOPTS_NO_EXCEPTIONS is not defined, just throw + throw T{text}; +#else + // Otherwise manually instantiate the exception, print what() to stderr, + // and exit + T exception{text}; + std::cerr << exception.what() << std::endl; + std::exit(EXIT_FAILURE); +#endif + } + + namespace values + { + namespace parser_tool + { + struct IntegerDesc + { + std::string negative = ""; + std::string base = ""; + std::string value = ""; + }; + struct ArguDesc { + std::string arg_name = ""; + bool grouping = false; + bool set_value = false; + std::string value = ""; + }; +#ifdef CXXOPTS_NO_REGEX + inline IntegerDesc SplitInteger(const std::string &text) + { + if (text.empty()) + { + throw_or_mimic(text); + } + IntegerDesc desc; + const char *pdata = text.c_str(); + if (*pdata == '-') + { + pdata += 1; + desc.negative = "-"; + } + if (strncmp(pdata, "0x", 2) == 0) + { + pdata += 2; + desc.base = "0x"; + } + if (*pdata != '\0') + { + desc.value = std::string(pdata); + } + else + { + throw_or_mimic(text); + } + return desc; + } + + inline bool IsTrueText(const std::string &text) + { + const char *pdata = text.c_str(); + if (*pdata == 't' || *pdata == 'T') + { + pdata += 1; + if (strncmp(pdata, "rue\0", 4) == 0) + { + return true; + } + } + else if (strncmp(pdata, "1\0", 2) == 0) + { + return true; + } + return false; + } + + inline bool IsFalseText(const std::string &text) + { + const char *pdata = text.c_str(); + if (*pdata == 'f' || *pdata == 'F') + { + pdata += 1; + if (strncmp(pdata, "alse\0", 5) == 0) + { + return true; + } + } + else if (strncmp(pdata, "0\0", 2) == 0) + { + return true; + } + return false; + } + + inline std::pair SplitSwitchDef(const std::string &text) + { + std::string short_sw, long_sw; + const char *pdata = text.c_str(); + if (isalnum(*pdata) && *(pdata + 1) == ',') { + short_sw = std::string(1, *pdata); + pdata += 2; + } + while (*pdata == ' ') { pdata += 1; } + if (isalnum(*pdata)) { + const char *store = pdata; + pdata += 1; + while (isalnum(*pdata) || *pdata == '-' || *pdata == '_') { + pdata += 1; + } + if (*pdata == '\0') { + long_sw = std::string(store, pdata - store); + } else { + throw_or_mimic(text); + } + } + return std::pair(short_sw, long_sw); + } + + inline ArguDesc ParseArgument(const char *arg, bool &matched) + { + ArguDesc argu_desc; + const char *pdata = arg; + matched = false; + if (strncmp(pdata, "--", 2) == 0) + { + pdata += 2; + if (isalnum(*pdata)) + { + argu_desc.arg_name.push_back(*pdata); + pdata += 1; + while (isalnum(*pdata) || *pdata == '-' || *pdata == '_') + { + argu_desc.arg_name.push_back(*pdata); + pdata += 1; + } + if (argu_desc.arg_name.length() > 1) + { + if (*pdata == '=') + { + argu_desc.set_value = true; + pdata += 1; + if (*pdata != '\0') + { + argu_desc.value = std::string(pdata); + } + matched = true; + } + else if (*pdata == '\0') + { + matched = true; + } + } + } + } + else if (strncmp(pdata, "-", 1) == 0) + { + pdata += 1; + argu_desc.grouping = true; + while (isalnum(*pdata)) + { + argu_desc.arg_name.push_back(*pdata); + pdata += 1; + } + matched = !argu_desc.arg_name.empty() && *pdata == '\0'; + } + return argu_desc; + } + +#else // CXXOPTS_NO_REGEX + + namespace + { + + std::basic_regex integer_pattern + ("(-)?(0x)?([0-9a-zA-Z]+)|((0x)?0)"); + std::basic_regex truthy_pattern + ("(t|T)(rue)?|1"); + std::basic_regex falsy_pattern + ("(f|F)(alse)?|0"); + + std::basic_regex option_matcher + ("--([[:alnum:]][-_[:alnum:]]+)(=(.*))?|-([[:alnum:]]+)"); + std::basic_regex option_specifier + ("(([[:alnum:]]),)?[ ]*([[:alnum:]][-_[:alnum:]]*)?"); + + } // namespace + + inline IntegerDesc SplitInteger(const std::string &text) + { + std::smatch match; + std::regex_match(text, match, integer_pattern); + + if (match.length() == 0) + { + throw_or_mimic(text); + } + + IntegerDesc desc; + desc.negative = match[1]; + desc.base = match[2]; + desc.value = match[3]; + + if (match.length(4) > 0) + { + desc.base = match[5]; + desc.value = "0"; + return desc; + } + + return desc; + } + + inline bool IsTrueText(const std::string &text) + { + std::smatch result; + std::regex_match(text, result, truthy_pattern); + return !result.empty(); + } + + inline bool IsFalseText(const std::string &text) + { + std::smatch result; + std::regex_match(text, result, falsy_pattern); + return !result.empty(); + } + + inline std::pair SplitSwitchDef(const std::string &text) + { + std::match_results result; + std::regex_match(text.c_str(), result, option_specifier); + if (result.empty()) + { + throw_or_mimic(text); + } + + const std::string& short_sw = result[2]; + const std::string& long_sw = result[3]; + + return std::pair(short_sw, long_sw); + } + + inline ArguDesc ParseArgument(const char *arg, bool &matched) + { + std::match_results result; + std::regex_match(arg, result, option_matcher); + matched = !result.empty(); + + ArguDesc argu_desc; + if (matched) { + argu_desc.arg_name = result[1].str(); + argu_desc.set_value = result[2].length() > 0; + argu_desc.value = result[3].str(); + if (result[4].length() > 0) + { + argu_desc.grouping = true; + argu_desc.arg_name = result[4].str(); + } + } + + return argu_desc; + } + +#endif // CXXOPTS_NO_REGEX +#undef CXXOPTS_NO_REGEX + } + + namespace detail + { + template + struct SignedCheck; + + template + struct SignedCheck + { + template + void + operator()(bool negative, U u, const std::string& text) + { + if (negative) + { + if (u > static_cast((std::numeric_limits::min)())) + { + throw_or_mimic(text); + } + } + else + { + if (u > static_cast((std::numeric_limits::max)())) + { + throw_or_mimic(text); + } + } + } + }; + + template + struct SignedCheck + { + template + void + operator()(bool, U, const std::string&) const {} + }; + + template + void + check_signed_range(bool negative, U value, const std::string& text) + { + SignedCheck::is_signed>()(negative, value, text); + } + } // namespace detail + + template + void + checked_negate(R& r, T&& t, const std::string&, std::true_type) + { + // if we got to here, then `t` is a positive number that fits into + // `R`. So to avoid MSVC C4146, we first cast it to `R`. + // See https://github.com/jarro2783/cxxopts/issues/62 for more details. + r = static_cast(-static_cast(t-1)-1); + } + + template + void + checked_negate(R&, T&&, const std::string& text, std::false_type) + { + throw_or_mimic(text); + } + + template + void + integer_parser(const std::string& text, T& value) + { + parser_tool::IntegerDesc int_desc = parser_tool::SplitInteger(text); + + using US = typename std::make_unsigned::type; + constexpr bool is_signed = std::numeric_limits::is_signed; + + const bool negative = int_desc.negative.length() > 0; + const uint8_t base = int_desc.base.length() > 0 ? 16 : 10; + const std::string & value_match = int_desc.value; + + US result = 0; + + for (char ch : value_match) + { + US digit = 0; + + if (ch >= '0' && ch <= '9') + { + digit = static_cast(ch - '0'); + } + else if (base == 16 && ch >= 'a' && ch <= 'f') + { + digit = static_cast(ch - 'a' + 10); + } + else if (base == 16 && ch >= 'A' && ch <= 'F') + { + digit = static_cast(ch - 'A' + 10); + } + else + { + throw_or_mimic(text); + } + + const US next = static_cast(result * base + digit); + if (result > next) + { + throw_or_mimic(text); + } + + result = next; + } + + detail::check_signed_range(negative, result, text); + + if (negative) + { + checked_negate(value, result, text, std::integral_constant()); + } + else + { + value = static_cast(result); + } + } + + template + void stringstream_parser(const std::string& text, T& value) + { + std::stringstream in(text); + in >> value; + if (!in) { + throw_or_mimic(text); + } + } + + template ::value>::type* = nullptr + > + void parse_value(const std::string& text, T& value) + { + integer_parser(text, value); + } + + inline + void + parse_value(const std::string& text, bool& value) + { + if (parser_tool::IsTrueText(text)) + { + value = true; + return; + } + + if (parser_tool::IsFalseText(text)) + { + value = false; + return; + } + + throw_or_mimic(text); + } + + inline + void + parse_value(const std::string& text, std::string& value) + { + value = text; + } + + // The fallback parser. It uses the stringstream parser to parse all types + // that have not been overloaded explicitly. It has to be placed in the + // source code before all other more specialized templates. + template ::value>::type* = nullptr + > + void + parse_value(const std::string& text, T& value) { + stringstream_parser(text, value); + } + + template + void + parse_value(const std::string& text, std::vector& value) + { + if (text.empty()) { + T v; + parse_value(text, v); + value.emplace_back(std::move(v)); + return; + } + std::stringstream in(text); + std::string token; + while(!in.eof() && std::getline(in, token, CXXOPTS_VECTOR_DELIMITER)) { + T v; + parse_value(token, v); + value.emplace_back(std::move(v)); + } + } + +#ifdef CXXOPTS_HAS_OPTIONAL + template + void + parse_value(const std::string& text, std::optional& value) + { + T result; + parse_value(text, result); + value = std::move(result); + } +#endif + + inline + void parse_value(const std::string& text, char& c) + { + if (text.length() != 1) + { + throw_or_mimic(text); + } + + c = text[0]; + } + + template + struct type_is_container + { + static constexpr bool value = false; + }; + + template + struct type_is_container> + { + static constexpr bool value = true; + }; + + template + class abstract_value : public Value + { + using Self = abstract_value; + + public: + abstract_value() + : m_result(std::make_shared()) + , m_store(m_result.get()) + { + } + + explicit abstract_value(T* t) + : m_store(t) + { + } + + ~abstract_value() override = default; + + abstract_value& operator=(const abstract_value&) = default; + + abstract_value(const abstract_value& rhs) + { + if (rhs.m_result) + { + m_result = std::make_shared(); + m_store = m_result.get(); + } + else + { + m_store = rhs.m_store; + } + + m_default = rhs.m_default; + m_implicit = rhs.m_implicit; + m_default_value = rhs.m_default_value; + m_implicit_value = rhs.m_implicit_value; + } + + void + parse(const std::string& text) const override + { + parse_value(text, *m_store); + } + + bool + is_container() const override + { + return type_is_container::value; + } + + void + parse() const override + { + parse_value(m_default_value, *m_store); + } + + bool + has_default() const override + { + return m_default; + } + + bool + has_implicit() const override + { + return m_implicit; + } + + std::shared_ptr + default_value(const std::string& value) override + { + m_default = true; + m_default_value = value; + return shared_from_this(); + } + + std::shared_ptr + implicit_value(const std::string& value) override + { + m_implicit = true; + m_implicit_value = value; + return shared_from_this(); + } + + std::shared_ptr + no_implicit_value() override + { + m_implicit = false; + return shared_from_this(); + } + + std::string + get_default_value() const override + { + return m_default_value; + } + + std::string + get_implicit_value() const override + { + return m_implicit_value; + } + + bool + is_boolean() const override + { + return std::is_same::value; + } + + const T& + get() const + { + if (m_store == nullptr) + { + return *m_result; + } + return *m_store; + } + + protected: + std::shared_ptr m_result{}; + T* m_store{}; + + bool m_default = false; + bool m_implicit = false; + + std::string m_default_value{}; + std::string m_implicit_value{}; + }; + + template + class standard_value : public abstract_value + { + public: + using abstract_value::abstract_value; + + CXXOPTS_NODISCARD + std::shared_ptr + clone() const override + { + return std::make_shared>(*this); + } + }; + + template <> + class standard_value : public abstract_value + { + public: + ~standard_value() override = default; + + standard_value() + { + set_default_and_implicit(); + } + + explicit standard_value(bool* b) + : abstract_value(b) + { + set_default_and_implicit(); + } + + std::shared_ptr + clone() const override + { + return std::make_shared>(*this); + } + + private: + + void + set_default_and_implicit() + { + m_default = true; + m_default_value = "false"; + m_implicit = true; + m_implicit_value = "true"; + } + }; + } // namespace values + + template + std::shared_ptr + value() + { + return std::make_shared>(); + } + + template + std::shared_ptr + value(T& t) + { + return std::make_shared>(&t); + } + + class OptionAdder; + + class OptionDetails + { + public: + OptionDetails + ( + std::string short_, + std::string long_, + String desc, + std::shared_ptr val + ) + : m_short(std::move(short_)) + , m_long(std::move(long_)) + , m_desc(std::move(desc)) + , m_value(std::move(val)) + , m_count(0) + { + m_hash = std::hash{}(m_long + m_short); + } + + OptionDetails(const OptionDetails& rhs) + : m_desc(rhs.m_desc) + , m_value(rhs.m_value->clone()) + , m_count(rhs.m_count) + { + } + + OptionDetails(OptionDetails&& rhs) = default; + + CXXOPTS_NODISCARD + const String& + description() const + { + return m_desc; + } + + CXXOPTS_NODISCARD + const Value& + value() const { + return *m_value; + } + + CXXOPTS_NODISCARD + std::shared_ptr + make_storage() const + { + return m_value->clone(); + } + + CXXOPTS_NODISCARD + const std::string& + short_name() const + { + return m_short; + } + + CXXOPTS_NODISCARD + const std::string& + long_name() const + { + return m_long; + } + + size_t + hash() const + { + return m_hash; + } + + private: + std::string m_short{}; + std::string m_long{}; + String m_desc{}; + std::shared_ptr m_value{}; + int m_count; + + size_t m_hash{}; + }; + + struct HelpOptionDetails + { + std::string s; + std::string l; + String desc; + bool has_default; + std::string default_value; + bool has_implicit; + std::string implicit_value; + std::string arg_help; + bool is_container; + bool is_boolean; + }; + + struct HelpGroupDetails + { + std::string name{}; + std::string description{}; + std::vector options{}; + }; + + class OptionValue + { + public: + void + parse + ( + const std::shared_ptr& details, + const std::string& text + ) + { + ensure_value(details); + ++m_count; + m_value->parse(text); + m_long_name = &details->long_name(); + } + + void + parse_default(const std::shared_ptr& details) + { + ensure_value(details); + m_default = true; + m_long_name = &details->long_name(); + m_value->parse(); + } + + void + parse_no_value(const std::shared_ptr& details) + { + m_long_name = &details->long_name(); + } + +#if defined(CXXOPTS_NULL_DEREF_IGNORE) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnull-dereference" +#endif + + CXXOPTS_NODISCARD + size_t + count() const noexcept + { + return m_count; + } + +#if defined(CXXOPTS_NULL_DEREF_IGNORE) +#pragma GCC diagnostic pop +#endif + + // TODO: maybe default options should count towards the number of arguments + CXXOPTS_NODISCARD + bool + has_default() const noexcept + { + return m_default; + } + + template + const T& + as() const + { + if (m_value == nullptr) { + throw_or_mimic( + m_long_name == nullptr ? "" : *m_long_name); + } + +#ifdef CXXOPTS_NO_RTTI + return static_cast&>(*m_value).get(); +#else + return dynamic_cast&>(*m_value).get(); +#endif + } + + private: + void + ensure_value(const std::shared_ptr& details) + { + if (m_value == nullptr) + { + m_value = details->make_storage(); + } + } + + + const std::string* m_long_name = nullptr; + // Holding this pointer is safe, since OptionValue's only exist in key-value pairs, + // where the key has the string we point to. + std::shared_ptr m_value{}; + size_t m_count = 0; + bool m_default = false; + }; + + class KeyValue + { + public: + KeyValue(std::string key_, std::string value_) + : m_key(std::move(key_)) + , m_value(std::move(value_)) + { + } + + CXXOPTS_NODISCARD + const std::string& + key() const + { + return m_key; + } + + CXXOPTS_NODISCARD + const std::string& + value() const + { + return m_value; + } + + template + T + as() const + { + T result; + values::parse_value(m_value, result); + return result; + } + + private: + std::string m_key; + std::string m_value; + }; + + using ParsedHashMap = std::unordered_map; + using NameHashMap = std::unordered_map; + + class ParseResult + { + public: + class Iterator + { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = KeyValue; + using difference_type = void; + using pointer = const KeyValue*; + using reference = const KeyValue&; + + Iterator() = default; + Iterator(const Iterator&) = default; + + Iterator(const ParseResult *pr, bool end=false) + : m_pr(pr) + , m_iter(end? pr->m_defaults.end(): pr->m_sequential.begin()) + { + } + + Iterator& operator++() + { + ++m_iter; + if(m_iter == m_pr->m_sequential.end()) + { + m_iter = m_pr->m_defaults.begin(); + return *this; + } + return *this; + } + + Iterator operator++(int) + { + Iterator retval = *this; + ++(*this); + return retval; + } + + bool operator==(const Iterator& other) const + { + return m_iter == other.m_iter; + } + + bool operator!=(const Iterator& other) const + { + return !(*this == other); + } + + const KeyValue& operator*() + { + return *m_iter; + } + + const KeyValue* operator->() + { + return m_iter.operator->(); + } + + private: + const ParseResult* m_pr; + std::vector::const_iterator m_iter; + }; + + ParseResult() = default; + ParseResult(const ParseResult&) = default; + + ParseResult(NameHashMap&& keys, ParsedHashMap&& values, std::vector sequential, + std::vector default_opts, std::vector&& unmatched_args) + : m_keys(std::move(keys)) + , m_values(std::move(values)) + , m_sequential(std::move(sequential)) + , m_defaults(std::move(default_opts)) + , m_unmatched(std::move(unmatched_args)) + { + } + + ParseResult& operator=(ParseResult&&) = default; + ParseResult& operator=(const ParseResult&) = default; + + Iterator + begin() const + { + return Iterator(this); + } + + Iterator + end() const + { + return Iterator(this, true); + } + + size_t + count(const std::string& o) const + { + auto iter = m_keys.find(o); + if (iter == m_keys.end()) + { + return 0; + } + + auto viter = m_values.find(iter->second); + + if (viter == m_values.end()) + { + return 0; + } + + return viter->second.count(); + } + + const OptionValue& + operator[](const std::string& option) const + { + auto iter = m_keys.find(option); + + if (iter == m_keys.end()) + { + throw_or_mimic(option); + } + + auto viter = m_values.find(iter->second); + + if (viter == m_values.end()) + { + throw_or_mimic(option); + } + + return viter->second; + } + + const std::vector& + arguments() const + { + return m_sequential; + } + + const std::vector& + unmatched() const + { + return m_unmatched; + } + + const std::vector& + defaults() const + { + return m_defaults; + } + + const std::string + arguments_string() const + { + std::string result; + for(const auto& kv: m_sequential) + { + result += kv.key() + " = " + kv.value() + "\n"; + } + for(const auto& kv: m_defaults) + { + result += kv.key() + " = " + kv.value() + " " + "(default)" + "\n"; + } + return result; + } + + private: + NameHashMap m_keys{}; + ParsedHashMap m_values{}; + std::vector m_sequential{}; + std::vector m_defaults{}; + std::vector m_unmatched{}; + }; + + struct Option + { + Option + ( + std::string opts, + std::string desc, + std::shared_ptr value = ::cxxopts::value(), + std::string arg_help = "" + ) + : opts_(std::move(opts)) + , desc_(std::move(desc)) + , value_(std::move(value)) + , arg_help_(std::move(arg_help)) + { + } + + std::string opts_; + std::string desc_; + std::shared_ptr value_; + std::string arg_help_; + }; + + using OptionMap = std::unordered_map>; + using PositionalList = std::vector; + using PositionalListIterator = PositionalList::const_iterator; + + class OptionParser + { + public: + OptionParser(const OptionMap& options, const PositionalList& positional, bool allow_unrecognised) + : m_options(options) + , m_positional(positional) + , m_allow_unrecognised(allow_unrecognised) + { + } + + ParseResult + parse(int argc, const char* const* argv); + + bool + consume_positional(const std::string& a, PositionalListIterator& next); + + void + checked_parse_arg + ( + int argc, + const char* const* argv, + int& current, + const std::shared_ptr& value, + const std::string& name + ); + + void + add_to_option(OptionMap::const_iterator iter, const std::string& option, const std::string& arg); + + void + parse_option + ( + const std::shared_ptr& value, + const std::string& name, + const std::string& arg = "" + ); + + void + parse_default(const std::shared_ptr& details); + + void + parse_no_value(const std::shared_ptr& details); + + private: + + void finalise_aliases(); + + const OptionMap& m_options; + const PositionalList& m_positional; + + std::vector m_sequential{}; + std::vector m_defaults{}; + bool m_allow_unrecognised; + + ParsedHashMap m_parsed{}; + NameHashMap m_keys{}; + }; + + class Options + { + public: + + explicit Options(std::string program, std::string help_string = "") + : m_program(std::move(program)) + , m_help_string(toLocalString(std::move(help_string))) + , m_custom_help("[OPTION...]") + , m_positional_help("positional parameters") + , m_show_positional(false) + , m_allow_unrecognised(false) + , m_width(76) + , m_tab_expansion(false) + , m_options(std::make_shared()) + { + } + + Options& + positional_help(std::string help_text) + { + m_positional_help = std::move(help_text); + return *this; + } + + Options& + custom_help(std::string help_text) + { + m_custom_help = std::move(help_text); + return *this; + } + + Options& + show_positional_help() + { + m_show_positional = true; + return *this; + } + + Options& + allow_unrecognised_options() + { + m_allow_unrecognised = true; + return *this; + } + + Options& + set_width(size_t width) + { + m_width = width; + return *this; + } + + Options& + set_tab_expansion(bool expansion=true) + { + m_tab_expansion = expansion; + return *this; + } + + ParseResult + parse(int argc, const char* const* argv); + + OptionAdder + add_options(std::string group = ""); + + void + add_options + ( + const std::string& group, + std::initializer_list