-
Notifications
You must be signed in to change notification settings - Fork 924
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
nvCOMP GZIP integration #16770
nvCOMP GZIP integration #16770
Changes from all commits
88194bf
66bb171
3c52671
f464fd2
a80c916
6fc5cf6
3560559
ca9ae5f
149b618
5bc0fb0
07d5fec
225c2ec
796dda2
c64aac1
01146fc
7f15529
e18c4ff
aa99f47
7847abf
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,6 +23,7 @@ | |
#include <cudf/utilities/error.hpp> | ||
|
||
#include <nvcomp/deflate.h> | ||
#include <nvcomp/gzip.h> | ||
#include <nvcomp/lz4.h> | ||
#include <nvcomp/snappy.h> | ||
#include <nvcomp/zstd.h> | ||
|
@@ -44,6 +45,8 @@ auto batched_decompress_get_temp_size_ex(compression_type compression, Args&&... | |
return nvcompBatchedLZ4DecompressGetTempSizeEx(std::forward<Args>(args)...); | ||
case compression_type::DEFLATE: | ||
return nvcompBatchedDeflateDecompressGetTempSizeEx(std::forward<Args>(args)...); | ||
case compression_type::GZIP: | ||
return nvcompBatchedGzipDecompressGetTempSizeEx(std::forward<Args>(args)...); | ||
default: CUDF_FAIL("Unsupported compression type"); | ||
} | ||
} | ||
|
@@ -73,6 +76,8 @@ auto batched_decompress_async(compression_type compression, Args&&... args) | |
case compression_type::DEFLATE: | ||
return nvcompBatchedDeflateDecompressAsync(std::forward<Args>(args)...); | ||
case compression_type::LZ4: return nvcompBatchedLZ4DecompressAsync(std::forward<Args>(args)...); | ||
case compression_type::GZIP: | ||
return nvcompBatchedGzipDecompressAsync(std::forward<Args>(args)...); | ||
default: CUDF_FAIL("Unsupported compression type"); | ||
} | ||
} | ||
|
@@ -84,6 +89,7 @@ std::string compression_type_name(compression_type compression) | |
case compression_type::ZSTD: return "Zstandard"; | ||
case compression_type::DEFLATE: return "Deflate"; | ||
case compression_type::LZ4: return "LZ4"; | ||
case compression_type::GZIP: return "GZIP"; | ||
} | ||
return "compression_type(" + std::to_string(static_cast<int>(compression)) + ")"; | ||
} | ||
|
@@ -359,8 +365,8 @@ std::optional<std::string> is_compression_disabled_impl(compression_type compres | |
return "nvCOMP use is disabled through the `LIBCUDF_NVCOMP_POLICY` environment variable."; | ||
} | ||
return std::nullopt; | ||
default: return "Unsupported compression type"; | ||
} | ||
return "Unsupported compression type"; | ||
} | ||
|
||
std::optional<std::string> is_compression_disabled(compression_type compression, | ||
|
@@ -396,7 +402,8 @@ std::optional<std::string> is_decompression_disabled_impl(compression_type compr | |
feature_status_parameters params) | ||
{ | ||
switch (compression) { | ||
case compression_type::DEFLATE: { | ||
case compression_type::DEFLATE: | ||
case compression_type::GZIP: { | ||
if (not params.are_all_integrations_enabled) { | ||
return "DEFLATE decompression is experimental, you can enable it through " | ||
"`LIBCUDF_NVCOMP_POLICY` environment variable."; | ||
|
@@ -447,6 +454,7 @@ std::optional<std::string> is_decompression_disabled(compression_type compressio | |
size_t required_alignment(compression_type compression) | ||
{ | ||
switch (compression) { | ||
case compression_type::GZIP: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why does this require the same alignment as There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. #16770 (comment) updated that comment to provide context |
||
case compression_type::DEFLATE: return nvcompDeflateRequiredAlignment; | ||
case compression_type::SNAPPY: return nvcompSnappyRequiredAlignment; | ||
case compression_type::ZSTD: return nvcompZstdRequiredAlignment; | ||
|
@@ -462,7 +470,7 @@ std::optional<size_t> compress_max_allowed_chunk_size(compression_type compressi | |
case compression_type::SNAPPY: return nvcompSnappyCompressionMaxAllowedChunkSize; | ||
case compression_type::ZSTD: return nvcompZstdCompressionMaxAllowedChunkSize; | ||
case compression_type::LZ4: return nvcompLZ4CompressionMaxAllowedChunkSize; | ||
default: return std::nullopt; | ||
default: CUDF_FAIL("Unsupported compression type"); | ||
} | ||
} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nvCOMP omitted
nvcompGzipRequiredAlignment
, so I could not follow the pattern we use for other formats.We can return the same value as deflate because it's the same compression format; the only difference is an additional header in GZIP.
The alternative would be to use a hard-coded number.