Skip to content

Commit

Permalink
polish
Browse files Browse the repository at this point in the history
  • Loading branch information
@henryzhx8
henryzhx8 committed Dec 28, 2024
1 parent 9d76a1e commit b216e85
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 196 deletions.
171 changes: 0 additions & 171 deletions core/common/CompressTools.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,141 +15,9 @@
#include "CompressTools.h"

#include <lz4/lz4.h>
#ifdef __ANDROID__
#include <zlib.h>
#else
#include <zlib/zlib.h>
#endif
#include <zstd/zstd.h>

#include <cstring>

#include "protobuf/sls/sls_logs.pb.h"

namespace logtail {

const int32_t ZSTD_DEFAULT_LEVEL = 1;

bool UncompressData(sls_logs::SlsCompressType compressType,
const std::string& src,
uint32_t rawSize,
std::string& dst) {
switch (compressType) {
case sls_logs::SLS_CMP_NONE:
dst = src;
return true;
case sls_logs::SLS_CMP_LZ4:
return UncompressLz4(src, rawSize, dst);
case sls_logs::SLS_CMP_DEFLATE:
return UncompressDeflate(src, rawSize, dst);
case sls_logs::SLS_CMP_ZSTD:
return UncompressZstd(src, rawSize, dst);
default:
return false;
}
}

bool CompressData(sls_logs::SlsCompressType compressType, const std::string& src, std::string& dst) {
switch (compressType) {
case sls_logs::SLS_CMP_NONE:
dst = src;
return true;
case sls_logs::SLS_CMP_LZ4:
return CompressLz4(src, dst);
case sls_logs::SLS_CMP_DEFLATE:
return CompressDeflate(src, dst);
case sls_logs::SLS_CMP_ZSTD:
return CompressZstd(src, dst, ZSTD_DEFAULT_LEVEL);
default:
return false;
}
}

bool CompressData(sls_logs::SlsCompressType compressType, const char* src, uint32_t size, std::string& dst) {
switch (compressType) {
case sls_logs::SLS_CMP_NONE: {
dst.assign(src, size);
return true;
}
case sls_logs::SLS_CMP_LZ4:
return CompressLz4(src, size, dst);
case sls_logs::SLS_CMP_DEFLATE:
return CompressDeflate(src, size, dst);
case sls_logs::SLS_CMP_ZSTD:
return CompressZstd(src, size, dst, ZSTD_DEFAULT_LEVEL);
default:
return false;
}
}

bool UncompressLz4(const std::string& src, const uint32_t rawSize, char* dst) {
uint32_t length = 0;
try {
length = LZ4_decompress_safe(src.c_str(), dst, src.length(), rawSize);
} catch (...) {
return false;
}
if (length != rawSize) {
return false;
}
return true;
}

bool UncompressLz4(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst) {
dst.resize(rawSize);
char* unCompressed = const_cast<char*>(dst.c_str());
uint32_t length = 0;
try {
length = LZ4_decompress_safe(srcPtr, unCompressed, srcSize, rawSize);
} catch (...) {
return false;
}
if (length != rawSize) {
return false;
}
return true;
}
bool CompressDeflate(const char* srcPtr, const uint32_t srcSize, std::string& dst) {
int64_t dstLen = compressBound(srcSize);
dst.resize(dstLen);
if (compress((Bytef*)(dst.c_str()), (uLongf*)&dstLen, (const Bytef*)srcPtr, srcSize) == Z_OK) {
dst.resize(dstLen);
return true;
}
return false;
}

bool CompressDeflate(const std::string& src, std::string& dst) {
int64_t dstLen = compressBound(src.size());
dst.resize(dstLen);
if (compress((Bytef*)(dst.c_str()), (uLongf*)&dstLen, (const Bytef*)(src.c_str()), src.size()) == Z_OK) {
dst.resize(dstLen);
return true;
}
return false;
}

bool UncompressDeflate(const char* srcPtr, const uint32_t srcSize, const int64_t rawSize, std::string& dst) {
static const int64_t MAX_UMCOMPRESS_SIZE = 128 * 1024 * 1024;
if (rawSize > MAX_UMCOMPRESS_SIZE) {
return false;
}
dst.resize(rawSize);
if (uncompress((Bytef*)(dst.c_str()), (uLongf*)&rawSize, (const Bytef*)(srcPtr), srcSize) != Z_OK) {
return false;
}
return true;
}


bool UncompressDeflate(const std::string& src, const int64_t rawSize, std::string& dst) {
return UncompressDeflate(src.c_str(), src.size(), rawSize, dst);
}


bool UncompressLz4(const std::string& src, const uint32_t rawSize, std::string& dst) {
return UncompressLz4(src.c_str(), src.length(), rawSize, dst);
}
bool CompressLz4(const char* srcPtr, const uint32_t srcSize, std::string& dst) {
uint32_t encodingSize = LZ4_compressBound(srcSize);
dst.resize(encodingSize);
Expand All @@ -169,43 +37,4 @@ bool CompressLz4(const std::string& src, std::string& dst) {
return CompressLz4(src.c_str(), src.length(), dst);
}

bool UncompressZstd(const std::string& src, const uint32_t rawSize, std::string& dst) {
return UncompressZstd(src.c_str(), src.length(), rawSize, dst);
}

bool UncompressZstd(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst) {
dst.resize(rawSize);
char* unCompressed = const_cast<char*>(dst.c_str());
uint32_t length = 0;
try {
length = ZSTD_decompress(unCompressed, rawSize, srcPtr, srcSize);
} catch (...) {
return false;
}
if (length != rawSize) {
return false;
}
return true;
}

bool CompressZstd(const char* srcPtr, const uint32_t srcSize, std::string& dst, int32_t level) {
uint32_t encodingSize = ZSTD_compressBound(srcSize);
dst.resize(encodingSize);
char* compressed = const_cast<char*>(dst.c_str());
try {
size_t const cmp_size = ZSTD_compress(compressed, encodingSize, srcPtr, srcSize, level);
if (ZSTD_isError(cmp_size)) {
return false;
}
dst.resize(cmp_size);
return true;
} catch (...) {
}
return false;
}

bool CompressZstd(const std::string& src, std::string& dst, int32_t level) {
return CompressZstd(src.c_str(), src.length(), dst, level);
}

} // namespace logtail
28 changes: 3 additions & 25 deletions core/common/CompressTools.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,14 @@
*/

#pragma once
#include <string>
#include <cstdint>
#include "protobuf/sls/sls_logs.pb.h"

namespace logtail {

extern const int32_t ZSTD_DEFAULT_LEVEL;

bool UncompressData(sls_logs::SlsCompressType compressType, const std::string& src, uint32_t rawSize, std::string& dst);

bool CompressData(sls_logs::SlsCompressType compressType, const std::string& src, std::string& dst);
bool CompressData(sls_logs::SlsCompressType compressType, const char* src, uint32_t size, std::string& dst);

bool UncompressDeflate(const std::string& src, const int64_t rawSize, std::string& dst);
bool UncompressDeflate(const char* srcPtr, const uint32_t srcSize, const int64_t rawSize, std::string& dst);
#include <cstdint>

bool CompressDeflate(const std::string& src, std::string& dst);
bool CompressDeflate(const char* srcPtr, const uint32_t srcSize, std::string& dst);
#include <string>

bool UncompressLz4(const std::string& src, const uint32_t rawSize, std::string& dst);
bool UncompressLz4(const std::string& src, const uint32_t rawSize, char* dst);
bool UncompressLz4(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst);
namespace logtail {

bool CompressLz4(const std::string& src, std::string& dst);
bool CompressLz4(const char* srcPtr, const uint32_t srcSize, std::string& dest);

bool UncompressZstd(const std::string& src, const uint32_t rawSize, std::string& dst);
bool UncompressZstd(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst);

bool CompressZstd(const char* srcPtr, const uint32_t srcSize, std::string& dst, int32_t level);
bool CompressZstd(const std::string& src, std::string& dst, int32_t level);

} // namespace logtail

0 comments on commit b216e85

Please sign in to comment.