From 59d814d611344114b8773685dfc3007df1b99b5d Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 31 Oct 2024 06:35:12 +0000 Subject: [PATCH 01/12] polish --- core/pipeline/Pipeline.cpp | 4 + core/plugin/flusher/sls/FlusherSLS.cpp | 4 + core/protobuf/sls/RawLog.cpp | 387 ------------------- core/protobuf/sls/RawLog.h | 126 ------ core/protobuf/sls/RawLogGroup.cpp | 222 ----------- core/protobuf/sls/RawLogGroup.h | 280 -------------- core/unittest/log_pb/CMakeLists.txt | 12 +- core/unittest/log_pb/PBUnittest.cpp | 405 -------------------- core/unittest/pipeline/PipelineUnittest.cpp | 2 + 9 files changed, 18 insertions(+), 1424 deletions(-) delete mode 100644 core/protobuf/sls/RawLog.cpp delete mode 100644 core/protobuf/sls/RawLog.h delete mode 100644 core/protobuf/sls/RawLogGroup.cpp delete mode 100644 core/protobuf/sls/RawLogGroup.h delete mode 100644 core/unittest/log_pb/PBUnittest.cpp diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index db517363f8..7be446e4cf 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -388,6 +388,10 @@ bool Pipeline::Send(vector&& groupList) { auto before = chrono::system_clock::now(); bool allSucceeded = true; for (auto& group : groupList) { + if (group.GetEvents().empty()) { + LOG_DEBUG(sLogger, ("empty event group", "discard")("config", mName)); + continue; + } auto res = mRouter.Route(group); for (auto& item : res) { if (item.first >= mFlushers.size()) { diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 9f5b4cd08b..6ba452f564 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -486,7 +486,11 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mPluginID, CompressType::LZ4); } +#ifdef __ENTERPRISE__ + mGroupSerializer = make_unique(this); +#else mGroupSerializer = make_unique(this); +#endif mGroupListSerializer = make_unique(this); // MaxSendRate diff --git a/core/protobuf/sls/RawLog.cpp b/core/protobuf/sls/RawLog.cpp deleted file mode 100644 index 359857ffc5..0000000000 --- a/core/protobuf/sls/RawLog.cpp +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "RawLog.h" -#include "common/Flags.h" -#include "logger/Logger.h" - -DEFINE_FLAG_INT32(raw_log_init_buffer_size, "", 256); - -namespace logtail { - -// 1+5( 1 ---> header; 5 ---> uint32) -#define INIT_LOG_SIZE_BYTES 6L - -/** - * Return the number of bytes required to store a variable-length unsigned - * 32-bit integer in base-128 varint encoding. - * - * \param v - * Value to encode. - * \return - * Number of bytes required. - */ -static inline size_t uint32_size(uint32_t v) { - if (v < (1UL << 7)) { - return 1; - } else if (v < (1UL << 14)) { - return 2; - } else if (v < (1UL << 21)) { - return 3; - } else if (v < (1UL << 28)) { - return 4; - } else { - return 5; - } -} - -/** - * Pack an unsigned 32-bit integer in base-128 varint encoding and return the - * number of bytes written, which must be 5 or less. - * - * \param value - * Value to encode. - * \param[out] out - * Packed value. - * \return - * Number of bytes written to `out`. - */ -static inline size_t uint32_pack(uint32_t value, uint8_t* out) { - unsigned rv = 0; - - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - } - } - } - } - /* assert: value<128 */ - out[rv++] = value; - return rv; -} - -static inline uint32_t parse_uint32(unsigned len, const uint8_t* data) { - uint32_t rv = data[0] & 0x7f; - if (len > 1) { - rv |= ((uint32_t)(data[1] & 0x7f) << 7); - if (len > 2) { - rv |= ((uint32_t)(data[2] & 0x7f) << 14); - if (len > 3) { - rv |= ((uint32_t)(data[3] & 0x7f) << 21); - if (len > 4) - rv |= ((uint32_t)(data[4]) << 28); - } - } - } - return rv; -} - -static unsigned scan_varint(unsigned len, const uint8_t* data) { - unsigned i; - if (len > 10) - len = 10; - for (i = 0; i < len; i++) - if ((data[i] & 0x80) == 0) - break; - if (i == len) - return 0; - return i + 1; -} - -RawLog* -RawLog::AddLogFull(uint32_t logTime, std::vector keys, boost::match_results subMathValues) { - // limit logTime's min value, ensure varint size is 5 - if (logTime < 463563523) { - logTime = 463563523; - } - - int32_t i = 0; - uint32_t logSize = 6; - int32_t pair_count = (int32_t)keys.size(); - for (; i < pair_count; ++i) { - auto keyLen = uint32_t(keys[i].size()); - auto valLen = uint32_t(subMathValues[i + 1].length()); - uint32_t contSize = uint32_size(keyLen) + uint32_size(valLen) + keyLen + valLen + 2u; - logSize += 1 + uint32_size(contSize) + contSize; - } - uint32_t headerSize = 1 + uint32_size(logSize); - int32_t deltaSize = INIT_LOG_SIZE_BYTES - headerSize; - uint32_t totalBufferSize = logSize + headerSize; - - auto* rawLog = new RawLog(); - rawLog->mRawLen = totalBufferSize + deltaSize; - rawLog->mRawBuffer = (uint8_t*)malloc(totalBufferSize + deltaSize); - rawLog->mMaxLen = totalBufferSize + deltaSize; - - uint8_t* buf = (uint8_t*)rawLog->mRawBuffer + deltaSize; - - *buf++ = 0x0A; - buf += uint32_pack(logSize, buf); - - // time - *buf++ = 0x08; - buf += uint32_pack(logTime, buf); - - // Content - // header - i = 0; - for (; i < pair_count; ++i) { - auto keyLen = uint32_t(keys[i].size()); - auto valLen = uint32_t(subMathValues[i + 1].length()); - *buf++ = 0x12; - buf += uint32_pack(uint32_size(keyLen) + uint32_size(valLen) + 2 + keyLen + valLen, buf); - *buf++ = 0x0A; - buf += uint32_pack(keyLen, buf); - memcpy(buf, keys[i].c_str(), keyLen); - buf += keyLen; - *buf++ = 0x12; - buf += uint32_pack(valLen, buf); - memcpy(buf, subMathValues[i + 1].first, valLen); - buf += valLen; - } - rawLog->mMallocDelta = deltaSize; - assert(buf - rawLog->mRawBuffer == totalBufferSize + rawLog->mMallocDelta); - rawLog->mNowBuffer = buf; - return rawLog; -} - -void RawLog::AddTime(uint32_t logTime) { - // limit logTime's min value, ensure varint size is 5 - if (logTime < 463563523) { - logTime = 463563523; - } - // 1 header and 5 time - if (mRawLen + 6 > mMaxLen) { - // reset log_now_buffer - adjustBuffer(6); - } - - // time - *mNowBuffer++ = 0x08; - mNowBuffer += uint32_pack(logTime, mNowBuffer); - mRawLen += 6; -} - -uint32_t RawLog::AddKeyValue(const char* key, size_t key_len, const char* value, size_t value_len) { - // sum total size - uint32_t kv_size - = sizeof(char) * (key_len + value_len) + uint32_size((uint32_t)key_len) + uint32_size((uint32_t)value_len) + 2; - kv_size += 1 + uint32_size(kv_size); - // ensure buffer size - if (mRawLen + kv_size > mMaxLen) { - // reset log_now_buffer - adjustBuffer(kv_size); - } - mRawLen += kv_size; - uint8_t* buf = mNowBuffer; - // key_value header - *buf++ = 0x12; - buf += uint32_pack(uint32_size(key_len) + uint32_size(value_len) + 2 + key_len + value_len, buf); - // key len - *buf++ = 0x0A; - buf += uint32_pack(key_len, buf); - // key - memcpy(buf, key, key_len); - buf += key_len; - // value len - *buf++ = 0x12; - buf += uint32_pack(value_len, buf); - // value - memcpy(buf, value, value_len); - buf += value_len; - mNowBuffer = (uint8_t*)buf; - return kv_size; -} - - -void RawLog::AddLogStart(uint32_t logTime) { - if (mRawLen + INIT_LOG_SIZE_BYTES > mMaxLen) { - adjustBuffer(INIT_LOG_SIZE_BYTES); - } - mRawLen = INIT_LOG_SIZE_BYTES; - mNowBuffer += INIT_LOG_SIZE_BYTES; - AddTime(logTime); -} - -void RawLog::AddLogDone() { - uint32_t log_size = mRawLen - INIT_LOG_SIZE_BYTES; - // check total size and uint32_size(total size) - - int32_t header_size = uint32_size(log_size) + 1; - - if (header_size != INIT_LOG_SIZE_BYTES) { - mMallocDelta = (int8_t)(INIT_LOG_SIZE_BYTES - header_size); - } - // set log header - uint8_t* buf = (mRawBuffer + mMallocDelta); - *buf++ = 0x0A; - uint32_pack(log_size, buf); -} - -void RawLog::adjustBuffer(uint32_t newLen) { - if (mRawBuffer == NULL) { - mRawBuffer = (uint8_t*)malloc(INT32_FLAG(raw_log_init_buffer_size)); - mNowBuffer = mRawBuffer; - mMaxLen = INT32_FLAG(raw_log_init_buffer_size); - return; - } - uint32_t new_buffer_len = mMaxLen << 1; - - if (new_buffer_len < mMaxLen + newLen) { - new_buffer_len = mMaxLen + newLen; - } - - mRawBuffer = (uint8_t*)realloc(mRawBuffer, new_buffer_len); - mMaxLen = new_buffer_len; - mNowBuffer = mRawBuffer + mRawLen; -} - -void RawLog::AppendToString(std::string* output) const { - output->append((const char*)mRawBuffer + mMallocDelta, mRawLen - mMallocDelta); -} - -bool RawLog::NextKeyValue( - RawLog::iterator& iter, const char*& key, uint32_t& keyLen, const char*& value, uint32_t& valueLen) { - if (iter.mNowOffset >= (int32_t)mRawLen) { - return false; - } - assert(mRawBuffer[iter.mNowOffset] == 0x12); - iter.mLastOffset = iter.mNowOffset; - // skip log content - ++iter.mNowOffset; - uint32_t keyValueLenSize = scan_varint(5, mRawBuffer + iter.mNowOffset); - iter.mNowOffset += keyValueLenSize; - // parse key - ++iter.mNowOffset; - uint32_t keyLenSize = scan_varint(5, mRawBuffer + iter.mNowOffset); - keyLen = parse_uint32(keyLenSize, mRawBuffer + iter.mNowOffset); - iter.mNowOffset += keyLenSize; - key = (const char*)mRawBuffer + iter.mNowOffset; - iter.mNowOffset += keyLen; - // parse value - ++iter.mNowOffset; - uint32_t valueLenSize = scan_varint(5, mRawBuffer + iter.mNowOffset); - valueLen = parse_uint32(valueLenSize, mRawBuffer + iter.mNowOffset); - iter.mNowOffset += valueLenSize; - value = (const char*)mRawBuffer + iter.mNowOffset; - iter.mNowOffset += valueLen; - return true; -} - -bool RawLog::DeleteKeyValue(RawLog::iterator& iter) { - if (iter.mLastOffset == 0 || iter.mLastOffset == iter.mNowOffset) { - return false; - } - int32_t deltaSize = iter.mLastOffset - iter.mNowOffset; - memmove(mRawBuffer + iter.mLastOffset, mRawBuffer + iter.mNowOffset, mRawLen - iter.mNowOffset); - iter.mNowOffset = iter.mLastOffset; - adjustLogSize(deltaSize); - mRawLen += deltaSize; - return true; -} - -bool RawLog::UpdateKeyValue( - RawLog::iterator& iter, const char* key, size_t key_len, const char* value, size_t value_len) { - if (iter.mLastOffset == 0) { - return false; - } - // sum total size - int32_t kv_size - = sizeof(char) * (key_len + value_len) + uint32_size((uint32_t)key_len) + uint32_size((uint32_t)value_len) + 2; - int32_t lastSize = iter.mNowOffset - iter.mLastOffset; - kv_size += 1 + uint32_size(kv_size); - // ensure buffer size - int32_t deltaSize = kv_size - lastSize; - if (deltaSize > 0 && mRawLen + kv_size > mMaxLen) { - // reset log_now_buffer - adjustBuffer(kv_size); - } - - // move - if (deltaSize != 0) { - memmove(mRawBuffer + iter.mLastOffset + kv_size, mRawBuffer + iter.mNowOffset, mRawLen - iter.mNowOffset); - adjustLogSize(deltaSize); - } - - mRawLen += deltaSize; - uint8_t* buf = mRawBuffer + iter.mLastOffset; - // key_value header - *buf++ = 0x12; - buf += uint32_pack(uint32_size(key_len) + uint32_size(value_len) + 2 + key_len + value_len, buf); - // key len - *buf++ = 0x0A; - buf += uint32_pack(key_len, buf); - // key - memcpy(buf, key, key_len); - buf += key_len; - // value len - *buf++ = 0x12; - buf += uint32_pack(value_len, buf); - // value - memcpy(buf, value, value_len); - buf += value_len; - mNowBuffer = (uint8_t*)buf; - iter.mNowOffset = iter.mLastOffset + kv_size; - return true; -} - -bool RawLog::AppendKeyValue(const char* key, size_t kenLen, const char* value, size_t valueLen) { - uint32_t deltaSize = AddKeyValue(key, kenLen, value, valueLen); - adjustLogSize(deltaSize); - return false; -} - -RawLog::iterator RawLog::GetIterator() const { - auto iter = iterator{}; - // skip delta and log header - uint32_t startLen = mMallocDelta + 1; - // skip log header size - startLen += scan_varint(5, mRawBuffer + startLen); - // skip time - startLen += 6; - iter.mNowOffset = startLen; - assert(mRawBuffer[startLen] == 0x12); - return iter; -} - -void RawLog::adjustLogSize(int32_t deltaLen) { - if (deltaLen == 0) { - return; - } - - uint32_t logLenSize = scan_varint(5, mRawBuffer + mMallocDelta + 1); - int32_t logSize = parse_uint32(logLenSize, mRawBuffer + mMallocDelta + 1); - int32_t newSize = logSize + deltaLen; - uint32_t newHeaderSize = uint32_size(newSize) + 1; - - mMallocDelta = (int8_t)(INIT_LOG_SIZE_BYTES - newHeaderSize); - - // set log header - uint8_t* buf = (mRawBuffer + mMallocDelta); - *buf++ = 0x0A; - uint32_pack(newSize, buf); -} - - -} // namespace logtail diff --git a/core/protobuf/sls/RawLog.h b/core/protobuf/sls/RawLog.h deleted file mode 100644 index 0b303655d9..0000000000 --- a/core/protobuf/sls/RawLog.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LOGTAIL_RAWLOG_H -#define LOGTAIL_RAWLOG_H - -#include -#include -#include - -namespace logtail { - -class RawLog { -public: - struct iterator { - int32_t mLastOffset = 0; - int32_t mNowOffset = 0; - }; - - RawLog() = default; - ~RawLog() { - if (mRawBuffer != NULL) { - free(mRawBuffer); - } - } - - static RawLog* - AddLogFull(uint32_t logTime, std::vector keys, boost::match_results subMathValues); - - void AddLogStart(uint32_t logTime); - - - void AddKeyValue(const std::string& key, const std::string& value) { - AddKeyValue(key.c_str(), key.size(), value.c_str(), value.size()); - } - - void AddKeyValue(const std::string& key, const char* value, size_t valueLen) { - AddKeyValue(key.c_str(), key.size(), value, valueLen); - } - - uint32_t AddKeyValue(const char* key, size_t kenLen, const char* value, size_t valueLen); - - void AddLogDone(); - - uint8_t* GetBuffer() { return mRawBuffer + mMallocDelta; } - - size_t GetBufferLength() { return mRawLen - mMallocDelta; } - - void AppendToString(std::string* output) const; - - /** - * init iterator - * @return - */ - iterator GetIterator() const; - - /** - * get next key value pair, return false if end of log - * @param iter - * @param key - * @param keyLen - * @param value - * @param valueLen - * @return - */ - bool NextKeyValue(iterator& iter, const char*& key, uint32_t& keyLen, const char*& value, uint32_t& valueLen); - - /** - * delete this key value pair, return false if iter is invalid - * @param iter - * @return - */ - bool DeleteKeyValue(iterator& iter); - - /** - * update this key value pair - * @param iter - * @param key - * @param kenLen - * @param value - * @param valueLen - * @return - */ - bool UpdateKeyValue(iterator& iter, const char* key, size_t kenLen, const char* value, size_t valueLen); - - /** - * append key value pair after end of log - * @param key - * @param kenLen - * @param value - * @param valueLen - * @return - */ - bool AppendKeyValue(const char* key, size_t kenLen, const char* value, size_t valueLen); - - -protected: - void adjustBuffer(uint32_t newLen); - void AddTime(uint32_t logTime); - void adjustLogSize(int32_t deltaLen); - - uint32_t mRawLen = 0; - uint32_t mMaxLen = 0; - uint8_t* mRawBuffer = NULL; - uint8_t* mNowBuffer = NULL; - int8_t mMallocDelta = 0; -}; - - -} // namespace logtail - - -#endif // LOGTAIL_RAWLOG_H diff --git a/core/protobuf/sls/RawLogGroup.cpp b/core/protobuf/sls/RawLogGroup.cpp deleted file mode 100644 index 9ce1215b7c..0000000000 --- a/core/protobuf/sls/RawLogGroup.cpp +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "RawLogGroup.h" - -namespace logtail { - -/** - * Return the number of bytes required to store a variable-length unsigned - * 32-bit integer in base-128 varint encoding. - * - * \param v - * Value to encode. - * \return - * Number of bytes required. - */ -static inline size_t uint32_size(uint32_t v) { - if (v < (1UL << 7)) { - return 1; - } else if (v < (1UL << 14)) { - return 2; - } else if (v < (1UL << 21)) { - return 3; - } else if (v < (1UL << 28)) { - return 4; - } else { - return 5; - } -} - -/** - * Pack an unsigned 32-bit integer in base-128 varint encoding and return the - * number of bytes written, which must be 5 or less. - * - * \param value - * Value to encode. - * \param[out] out - * Packed value. - * \return - * Number of bytes written to `out`. - */ -static inline size_t uint32_pack(uint32_t value, std::string* output) { - unsigned rv = 1; - - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - } - } - } - } - /* assert: value<128 */ - output->push_back(value); - return rv; -} - - -int RawLogGroup::logtags_size() const { - return (int)logtags_.size(); -} - -void RawLogGroup::clear_logtags() { - logtags_.clear(); -} - -const LogTag& RawLogGroup::logtags(int index) const { - return logtags_[index]; -} - -LogTag* RawLogGroup::mutable_logtags(int index) { - return &logtags_[index]; -} - -void RawLogGroup::add_logtags(const std::string& key, const std::string& value) { - logtags_.push_back(::std::move(LogTag(key, value))); -} - -void RawLogGroup::add_logtags(const std::string& key, std::string&& value) { - logtags_.push_back(LogTag(key, ::std::move(value))); -} - -std::vector* RawLogGroup::mutable_logtags() { - return &logtags_; -} - -const std::vector& RawLogGroup::logtags() const { - return logtags_; -} - -RawLogGroup::~RawLogGroup() { - for (size_t size = 0; size < rawlogs_.size(); ++size) { - delete rawlogs_[size]; - } -} - -int RawLogGroup::logs_size() const { - return (int)rawlogs_.size(); -} - -void RawLogGroup::clear_logs() { - for (size_t size = 0; size < rawlogs_.size(); ++size) { - delete rawlogs_[size]; - } - rawlogs_.clear(); -} - -const RawLog& RawLogGroup::logs(int index) const { - return *rawlogs_[index]; -} - -RawLog* RawLogGroup::mutable_logs(int index) { - return rawlogs_[index]; -} - -void RawLogGroup::add_logs(RawLog* log) { - rawlogs_.push_back(log); -} - -std::vector* RawLogGroup::mutable_logs() { - return &rawlogs_; -} - -const std::vector& RawLogGroup::logs() { - return rawlogs_; -} - -bool RawLogGroup::SerializeToString(std::string* output) const { - output->clear(); - return AppendToString(output); -} - -bool RawLogGroup::AppendToString(std::string* output) const { - if (rawlogs_.empty()) { - return false; - } - pack_logs(output); - pack_others(output); - pack_logtags(output); - return true; -} - -void RawLogGroup::pack_logs(std::string* output) const { - for (size_t size = 0; size < rawlogs_.size(); ++size) { - RawLog* log = rawlogs_[size]; - log->AppendToString(output); - } -} - -void RawLogGroup::pack_logtags(std::string* output) const { - for (size_t size = 0; size < logtags_.size(); ++size) { - const LogTag& logTag = logtags_[size]; - - // use only 1 malloc - size_t k_len = logTag.first.size(); - size_t v_len = logTag.second.size(); - const char* k = logTag.first.c_str(); - const char* v = logTag.second.c_str(); - uint32_t tag_size - = sizeof(char) * (k_len + v_len) + uint32_size((uint32_t)k_len) + uint32_size((uint32_t)v_len) + 2; - output->push_back(0x32); - uint32_pack(tag_size, output); - output->push_back(0x0A); - uint32_pack((uint32_t)k_len, output); - output->append(k, k_len); - output->push_back(0x12); - uint32_pack((uint32_t)v_len, output); - output->append(v, v_len); - } -} - -void RawLogGroup::pack_others(std::string* output) const { - if (has_category()) { - output->push_back(0x12); - uint32_pack((uint32_t)category_.size(), output); - output->append(category_); - } - - if (has_topic()) { - output->push_back(0x1A); - uint32_pack((uint32_t)topic_.size(), output); - output->append(topic_); - } - - if (has_source()) { - output->push_back(0x22); - uint32_pack((uint32_t)source_.size(), output); - output->append(source_); - } - - if (has_machineuuid()) { - output->push_back(0x2A); - uint32_pack((uint32_t)machineuuid_.size(), output); - output->append(machineuuid_); - } -} - - -} // namespace logtail \ No newline at end of file diff --git a/core/protobuf/sls/RawLogGroup.h b/core/protobuf/sls/RawLogGroup.h deleted file mode 100644 index cf5a357f11..0000000000 --- a/core/protobuf/sls/RawLogGroup.h +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LOGTAIL_RAWLOGGROUP_H -#define LOGTAIL_RAWLOGGROUP_H - -#include - -namespace logtail { - -typedef std::pair LogTag; - -class RawLogGroup { -public: - RawLogGroup() = default; - ~RawLogGroup(); - - int logs_size() const; - void clear_logs(); - const RawLog& logs(int index) const; - RawLog* mutable_logs(int index); - void add_logs(RawLog* log); - std::vector* mutable_logs(); - const std::vector& logs(); - - // repeated .sls_logs.LogTag LogTags = 6; - int logtags_size() const; - void clear_logtags(); - const LogTag& logtags(int index) const; - LogTag* mutable_logtags(int index); - void add_logtags(const std::string& key, const std::string& value); - void add_logtags(const std::string& key, std::string&& value); - std::vector* mutable_logtags(); - const std::vector& logtags() const; - - // optional string Category = 2; 0x12 - bool has_category() const; - void clear_category(); - const ::std::string& category() const; - void set_category(const ::std::string& value); - void set_category(::std::string&& value); - void set_category(const char* value, size_t size); - ::std::string* mutable_category(); - - // optional string Topic = 3; 0x1A - bool has_topic() const; - void clear_topic(); - const ::std::string& topic() const; - void set_topic(const ::std::string& value); - void set_topic(::std::string&& value); - void set_topic(const char* value, size_t size); - ::std::string* mutable_topic(); - - // optional string Source = 4; 0x22; - bool has_source() const; - void clear_source(); - const ::std::string& source() const; - void set_source(const ::std::string& value); - void set_source(::std::string&& value); - void set_source(const char* value, size_t size); - ::std::string* mutable_source(); - - // optional string MachineUUID = 5; 0x2A - bool has_machineuuid() const; - void clear_machineuuid(); - static const int kMachineUUIDFieldNumber = 5; - const ::std::string& machineuuid() const; - void set_machineuuid(const ::std::string& value); - void set_machineuuid(::std::string&& value); - void set_machineuuid(const char* value, size_t size); - ::std::string* mutable_machineuuid(); - - // Serialize the message and store it in the given string. All required - // fields must be set. - bool SerializeToString(std::string* output) const; - // Like SerializeToString(), but appends to the data to the string's existing - // contents. All required fields must be set. - bool AppendToString(std::string* output) const; - -private: - void set_has_category(); - void clear_has_category(); - void set_has_topic(); - void clear_has_topic(); - void set_has_source(); - void clear_has_source(); - void set_has_machineuuid(); - void clear_has_machineuuid(); - - void pack_logs(std::string* output) const; - void pack_logtags(std::string* output) const; - void pack_others(std::string* output) const; - - - uint64_t _has_bits_ = 0; - std::string category_; - std::string topic_; - std::string source_; - std::string machineuuid_; - std::vector logtags_; - std::vector rawlogs_; -}; - -// optional string Category = 2; -inline bool RawLogGroup::has_category() const { - return (_has_bits_ & 0x00000001u) != 0; -} -inline void RawLogGroup::set_has_category() { - _has_bits_ |= 0x00000001u; -} -inline void RawLogGroup::clear_has_category() { - _has_bits_ &= ~0x00000001u; -} -inline void RawLogGroup::clear_category() { - category_.clear(); - clear_has_category(); -} -inline const ::std::string& RawLogGroup::category() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.Category) - return category_; -} -inline void RawLogGroup::set_category(const ::std::string& value) { - set_has_category(); - category_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.Category) -} -inline void RawLogGroup::set_category(::std::string&& value) { - set_has_category(); - category_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.Category) -} -inline void RawLogGroup::set_category(const char* value, size_t size) { - set_has_category(); - category_ = ::std::string(reinterpret_cast(value), size); - ; - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.Category) -} -inline ::std::string* RawLogGroup::mutable_category() { - set_has_category(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.Category) - return &category_; - ; -} - -// optional string Topic = 3; -inline bool RawLogGroup::has_topic() const { - return (_has_bits_ & 0x00000002u) != 0; -} -inline void RawLogGroup::set_has_topic() { - _has_bits_ |= 0x00000002u; -} -inline void RawLogGroup::clear_has_topic() { - _has_bits_ &= ~0x00000002u; -} -inline void RawLogGroup::clear_topic() { - topic_.clear(); - clear_has_topic(); -} -inline const ::std::string& RawLogGroup::topic() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.Topic) - return topic_; -} -inline void RawLogGroup::set_topic(const ::std::string& value) { - set_has_topic(); - topic_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.Topic) -} -inline void RawLogGroup::set_topic(::std::string&& value) { - set_has_topic(); - topic_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.Topic) -} -inline void RawLogGroup::set_topic(const char* value, size_t size) { - set_has_topic(); - topic_ = ::std::string(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.Topic) -} -inline ::std::string* RawLogGroup::mutable_topic() { - set_has_topic(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.Topic) - return &topic_; - ; -} - -// optional string Source = 4; -inline bool RawLogGroup::has_source() const { - return (_has_bits_ & 0x00000004u) != 0; -} -inline void RawLogGroup::set_has_source() { - _has_bits_ |= 0x00000004u; -} -inline void RawLogGroup::clear_has_source() { - _has_bits_ &= ~0x00000004u; -} -inline void RawLogGroup::clear_source() { - source_.clear(); - clear_has_source(); -} -inline const ::std::string& RawLogGroup::source() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.Source) - return source_; -} -inline void RawLogGroup::set_source(const ::std::string& value) { - set_has_source(); - source_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.Source) -} -inline void RawLogGroup::set_source(::std::string&& value) { - set_has_source(); - source_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.Source) -} -inline void RawLogGroup::set_source(const char* value, size_t size) { - set_has_source(); - source_ = ::std::string(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.Source) -} -inline ::std::string* RawLogGroup::mutable_source() { - set_has_source(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.Source) - return &source_; - ; -} - -// optional string MachineUUID = 5; -inline bool RawLogGroup::has_machineuuid() const { - return (_has_bits_ & 0x00000008u) != 0; -} -inline void RawLogGroup::set_has_machineuuid() { - _has_bits_ |= 0x00000008u; -} -inline void RawLogGroup::clear_has_machineuuid() { - _has_bits_ &= ~0x00000008u; -} -inline void RawLogGroup::clear_machineuuid() { - machineuuid_.clear(); - clear_has_machineuuid(); -} -inline const ::std::string& RawLogGroup::machineuuid() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.MachineUUID) - return machineuuid_; -} -inline void RawLogGroup::set_machineuuid(const ::std::string& value) { - set_has_machineuuid(); - machineuuid_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.MachineUUID) -} -inline void RawLogGroup::set_machineuuid(::std::string&& value) { - set_has_machineuuid(); - machineuuid_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.MachineUUID) -} -inline void RawLogGroup::set_machineuuid(const char* value, size_t size) { - set_has_machineuuid(); - machineuuid_ = ::std::string(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.MachineUUID) -} -inline ::std::string* RawLogGroup::mutable_machineuuid() { - set_has_machineuuid(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.MachineUUID) - return &machineuuid_; - ; -} - -} // namespace logtail - -#endif // LOGTAIL_RAWLOGGROUP_H diff --git a/core/unittest/log_pb/CMakeLists.txt b/core/unittest/log_pb/CMakeLists.txt index 1941bc0a39..6a7ad3019c 100644 --- a/core/unittest/log_pb/CMakeLists.txt +++ b/core/unittest/log_pb/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2022 iLogtail Authors +# Copyright 2024 iLogtail Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +15,12 @@ cmake_minimum_required(VERSION 3.22) project(log_pb_unittest) -add_executable(log_pb_unittest PBUnittest.cpp) -target_link_libraries(log_pb_unittest ${UT_BASE_TARGET}) +if (ENABLE_ENTERPRISE) + add_executable(log_group_serializer_unittest LogGroupSerializerUnittest.cpp) + target_link_libraries(log_group_serializer_unittest ${UT_BASE_TARGET}) +endif () include(GoogleTest) -gtest_discover_tests(log_pb_unittest) +if (ENABLE_ENTERPRISE) + gtest_discover_tests(log_group_serializer_unittest) +endif () diff --git a/core/unittest/log_pb/PBUnittest.cpp b/core/unittest/log_pb/PBUnittest.cpp deleted file mode 100644 index 0e60ffcb96..0000000000 --- a/core/unittest/log_pb/PBUnittest.cpp +++ /dev/null @@ -1,405 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "unittest/Unittest.h" -#include "protobuf/sls/RawLogGroup.h" -#include "protobuf/sls/sls_logs.pb.h" - -using namespace std; -using namespace sls_logs; - -namespace logtail { - -class PBUnittest : public ::testing::Test { -public: - string longLogValue10 = string(10 * 1024, 'c'); - string longLogValue100 = string(100 * 1024, 'd'); - string longLogValue1000 = string(1000 * 1024, 'e'); - string longLogValue10000 = string(10000 * 1024, 'f'); - - static void SetUpTestCase() // void Setup(); - {} - static void TearDownTestCase() // void CleanUp(); - {} - - void TestFullWrite() { - boost::match_results what; - boost::regex reg("(\\w+) (\\w+) (\\w+) (\\w+)"); - boost::regex_match("1 234 567 890xyz", what, reg, boost::match_default); - vector keys; - keys.push_back("key1"); - keys.push_back("key2"); - keys.push_back("key3"); - keys.push_back(""); - - auto now = GetCurrentLogtailTime(); - RawLog* rawLog = RawLog::AddLogFull(now.tv_sec, keys, what); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("1"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value("234"); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("567"); - kv = log->add_contents(); - kv->set_key(""); - kv->set_value("890xyz"); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog->AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - - EXPECT_EQ(rawLogStr, logStr); - - delete rawLog; - } - - void TestNormalWrite() { - RawLog rawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog.AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - } - - void TestBigLog() { - RawLog rawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), longLogValue100.c_str(), longLogValue100.size()); - rawLog.AddKeyValue("key3", strlen("key3"), longLogValue10.c_str(), longLogValue10.size()); - rawLog.AddKeyValue("key4", strlen("key4"), longLogValue1000.c_str(), longLogValue1000.size()); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(longLogValue100); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value(longLogValue10); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value(longLogValue1000); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog.AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - } - - void TestIterator() { - RawLog rawLog; - int32_t nowTime = time(NULL); - rawLog.AddLogStart(nowTime); - rawLog.AddKeyValue("key1", strlen("key1"), "value1", strlen("value1")); - rawLog.AddKeyValue("key2", strlen("key2"), "value2", strlen("value2")); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value4", strlen("value4")); - rawLog.AddLogDone(); - - RawLog::iterator iter = rawLog.GetIterator(); - const char* key = NULL; - uint32_t keyLen = 0; - const char* value = NULL; - uint32_t valueLen = 0; - int i = 0; - while (rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)) { - ++i; - string keyStr(key, keyLen); - string valueStr(value, valueLen); - string expectKey = "key" + std::to_string(i); - string expectValue = "value" + std::to_string(i); - EXPECT_EQ(keyStr, expectKey); - EXPECT_EQ(valueStr, expectValue); - } - } - - void TestUpdateDeleteAppend() { - RawLog rawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value1", strlen("value1")); - rawLog.AddKeyValue("k", strlen("k"), "value", strlen("value")); - rawLog.AddKeyValue("k", strlen("k"), "value", strlen("value")); - rawLog.AddKeyValue("k", strlen("k"), "value", strlen("value")); - rawLog.AddLogDone(); - - RawLog::iterator iter = rawLog.GetIterator(); - const char* key = NULL; - uint32_t keyLen = 0; - const char* value = NULL; - uint32_t valueLen = 0; - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - string keyStr(key, keyLen); - string valueStr(value, valueLen); - EXPECT_EQ(keyStr, string("key1")); - EXPECT_EQ(valueStr, string("value1")); - } - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.UpdateKeyValue(iter, "key2", strlen("key2"), "value2", strlen("value2")); - } - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.DeleteKeyValue(iter); - } - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.UpdateKeyValue(iter, "key3", strlen("key3"), "value3", strlen("value3")); - } - EXPECT_FALSE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.AppendKeyValue("key4", strlen("key4"), "value4", strlen("value4")); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value1"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value("value2"); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value4"); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog.AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - } - - void TestLogGroup() { - RawLog* pRawLog = new RawLog(); - RawLog& rawLog = *pRawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - - - RawLogGroup* pRawLogGroup = new RawLogGroup; - pRawLogGroup->set_source("192.168.1.1"); - loggroup.set_source("192.168.1.1"); - - pRawLogGroup->set_category("logstore"); - loggroup.set_category("logstore"); - - pRawLogGroup->set_machineuuid("123-456"); - loggroup.set_machineuuid("123-456"); - - pRawLogGroup->set_topic("topic"); - loggroup.set_topic("topic"); - - pRawLogGroup->add_logtags("tagkey1", "tagvalue1"); - pRawLogGroup->add_logtags("tagkey2", "tagvalue2"); - - auto tag1 = loggroup.add_logtags(); - tag1->set_key("tagkey1"); - tag1->set_value("tagvalue1"); - - auto tag2 = loggroup.add_logtags(); - tag2->set_key("tagkey2"); - tag2->set_value("tagvalue2"); - - string rawLogStr; - pRawLogGroup->add_logs(pRawLog); - pRawLogGroup->AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - delete pRawLogGroup; - } - - void TestNoOptionLogGroup() { - RawLog* pRawLog = new RawLog(); - RawLog& rawLog = *pRawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - - - RawLogGroup* pRawLogGroup = new RawLogGroup; - pRawLogGroup->set_source("192.168.1.1"); - loggroup.set_source("192.168.1.1"); - - pRawLogGroup->set_topic("topic"); - loggroup.set_topic("topic"); - - pRawLogGroup->add_logtags("tagkey1", "tagvalue1"); - pRawLogGroup->add_logtags("tagkey2", "tagvalue2"); - - auto tag1 = loggroup.add_logtags(); - tag1->set_key("tagkey1"); - tag1->set_value("tagvalue1"); - - auto tag2 = loggroup.add_logtags(); - tag2->set_key("tagkey2"); - tag2->set_value("tagvalue2"); - - string rawLogStr; - pRawLogGroup->add_logs(pRawLog); - pRawLogGroup->AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - delete pRawLogGroup; - } - - void TestMultiLog() { - auto now = GetCurrentLogtailTime(); - string rawLogStr; - auto c1 = clock(); - for (int i = 0; i < 1000000; ++i) { - RawLog rawLog; - - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - rawLogStr.clear(); - rawLog.AppendToString(&rawLogStr); - } - auto c2 = clock(); - string logStr; - for (int i = 0; i < 1000000; ++i) { - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - logStr.clear(); - loggroup.AppendToString(&logStr); - } - auto c3 = clock(); - EXPECT_GT((c3 - c2) / 5, c2 - c1); - printf("%d %d \n", (int)(c3 - c2), (int)(c2 - c1)); - EXPECT_EQ(rawLogStr, logStr); - } -}; - -APSARA_UNIT_TEST_CASE(PBUnittest, TestFullWrite, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestNormalWrite, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestIterator, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestUpdateDeleteAppend, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestBigLog, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestLogGroup, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestNoOptionLogGroup, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestMultiLog, 0); - -} // namespace logtail - - -int main(int argc, char** argv) { - logtail::Logger::Instance().InitGlobalLoggers(); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index c77a1e1a7a..95fe13faee 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2762,6 +2762,7 @@ void PipelineUnittest::TestSend() const { { // all valid vector group; + group.back().AddLogEvent(); group.emplace_back(make_shared()); APSARA_TEST_TRUE(pipeline.Send(std::move(group))); } @@ -2771,6 +2772,7 @@ void PipelineUnittest::TestSend() const { = false; vector group; group.emplace_back(make_shared()); + group.back().AddLogEvent(); APSARA_TEST_FALSE(pipeline.Send(std::move(group))); const_cast(static_cast(pipeline.mFlushers[0]->GetPlugin()))->mIsValid = true; From 08e2df9ad0248d4e9c4a7ba5d1f7938ccc3090f9 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 31 Oct 2024 06:54:32 +0000 Subject: [PATCH 02/12] polish --- core/unittest/serializer/CMakeLists.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/unittest/serializer/CMakeLists.txt b/core/unittest/serializer/CMakeLists.txt index b471e3b81a..6fb442365f 100644 --- a/core/unittest/serializer/CMakeLists.txt +++ b/core/unittest/serializer/CMakeLists.txt @@ -18,9 +18,18 @@ project(serializer_unittest) add_executable(serializer_unittest SerializerUnittest.cpp) target_link_libraries(serializer_unittest ${UT_BASE_TARGET}) -add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) -target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) +if (ENABLE_ENTERPRISE) + add_executable(enterprise_sls_serializer_unittest EnterpriseSLSSerializerUnittest.cpp) + target_link_libraries(enterprise_sls_serializer_unittest ${UT_BASE_TARGET}) +else () + add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) + target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) +endif () include(GoogleTest) gtest_discover_tests(serializer_unittest) -gtest_discover_tests(sls_serializer_unittest) +if (ENABLE_ENTERPRISE) + gtest_discover_tests(enterprise_sls_serializer_unittest) +else () + gtest_discover_tests(sls_serializer_unittest) +endif () From d3dfeca38589f3869854d49904008b7d04bc5b98 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 31 Oct 2024 07:28:16 +0000 Subject: [PATCH 03/12] polish --- core/unittest/pipeline/PipelineUnittest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 95fe13faee..1cf92420d5 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2762,8 +2762,8 @@ void PipelineUnittest::TestSend() const { { // all valid vector group; - group.back().AddLogEvent(); group.emplace_back(make_shared()); + group.back().AddLogEvent(); APSARA_TEST_TRUE(pipeline.Send(std::move(group))); } { From b7fe757aa1702980e76bda193f23ccd1b7871b03 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 31 Oct 2024 07:33:38 +0000 Subject: [PATCH 04/12] polish --- core/plugin/flusher/sls/FlusherSLS.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index c9afca958a..16019ac225 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -30,15 +30,19 @@ #include "pipeline/batch/Batcher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" #include "pipeline/plugin/interface/HttpFlusher.h" +#ifdef __ENTERPRISE__ +#include "pipeline/serializer/EnterpriseSLSSerializer.h" +#else #include "pipeline/serializer/SLSSerializer.h" +#endif #include "protobuf/sls/sls_logs.pb.h" namespace logtail { class FlusherSLS : public HttpFlusher { public: - - static std::shared_ptr GetLogstoreConcurrencyLimiter(const std::string& project, const std::string& logstore); + static std::shared_ptr GetLogstoreConcurrencyLimiter(const std::string& project, + const std::string& logstore); static std::shared_ptr GetProjectConcurrencyLimiter(const std::string& project); static std::shared_ptr GetRegionConcurrencyLimiter(const std::string& region); static void ClearInvalidConcurrencyLimiters(); From bf8b2afe5688fd6bd33b3b903ea217d5db62fb77 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 31 Oct 2024 08:53:41 +0000 Subject: [PATCH 05/12] polish --- core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f46debed17..7d428d545e 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -157,12 +157,12 @@ endforeach (DIR_NAME) if (ENABLE_ENTERPRISE) # remove several files in shennong/sdk list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/shennong/sdk/sample.cpp) + list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/serializer/SLSSerializer.cpp) endif() # remove several files in go_pipeline list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h) - # add provider add_subdirectory("${PROVIDER_PATH}" "${CMAKE_BINARY_DIR}/provider") From 21e70f822650ea8b87d40112bdbcc903d397c0c2 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 1 Nov 2024 12:36:23 +0000 Subject: [PATCH 06/12] polish --- core/CMakeLists.txt | 1 - core/pipeline/serializer/SLSSerializer.cpp | 189 +++++++------ core/plugin/flusher/sls/FlusherSLS.cpp | 4 - core/plugin/flusher/sls/FlusherSLS.h | 4 - core/protobuf/sls/LogGroupSerializer.cpp | 259 ++++++++++++++++++ core/protobuf/sls/LogGroupSerializer.h | 64 +++++ core/unittest/log_pb/CMakeLists.txt | 10 +- .../log_pb/LogGroupSerializerUnittest.cpp | 103 +++++++ core/unittest/serializer/CMakeLists.txt | 15 +- 9 files changed, 540 insertions(+), 109 deletions(-) create mode 100644 core/protobuf/sls/LogGroupSerializer.cpp create mode 100644 core/protobuf/sls/LogGroupSerializer.h create mode 100644 core/unittest/log_pb/LogGroupSerializerUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 7d428d545e..9c9d7cffab 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -157,7 +157,6 @@ endforeach (DIR_NAME) if (ENABLE_ENTERPRISE) # remove several files in shennong/sdk list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/shennong/sdk/sample.cpp) - list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/serializer/SLSSerializer.cpp) endif() # remove several files in go_pipeline diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index f2a655869f..e43ec49104 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -12,25 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/serializer/SLSSerializer.h" +#include "pipeline/serializer/EnterpriseSLSSerializer.h" -#include "application/Application.h" #include "common/Flags.h" -#include "common/TimeUtil.h" #include "common/compression/CompressType.h" #include "plugin/flusher/sls/FlusherSLS.h" - +#include "protobuf/sls/LogGroupSerializer.h" DEFINE_FLAG_INT32(max_send_log_group_size, "bytes", 10 * 1024 * 1024); -const std::string METRIC_RESERVED_KEY_NAME = "__name__"; -const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; -const std::string METRIC_RESERVED_KEY_VALUE = "__value__"; -const std::string METRIC_RESERVED_KEY_TIME_NANO = "__time_nano__"; - -const std::string METRIC_LABELS_SEPARATOR = "|"; -const std::string METRIC_LABELS_KEY_VALUE_SEPARATOR = "#$#"; - using namespace std; namespace logtail { @@ -61,88 +51,125 @@ bool Serializer>::DoSerialize(vector()) { - const auto& logEvent = e.Cast(); - auto log = logGroup.add_logs(); - for (const auto& kv : logEvent) { - auto contPtr = log->add_contents(); - contPtr->set_key(kv.first.to_string()); - contPtr->set_value(kv.second.to_string()); - } - log->set_time(logEvent.GetTimestamp()); - if (mFlusher->GetContext().GetGlobalConfig().mEnableTimestampNanosecond - && logEvent.GetTimestampNanosecond()) { - log->set_time_ns(logEvent.GetTimestampNanosecond().value()); - } - } else if (e.Is()) { - const auto& metricEvent = e.Cast(); - if (metricEvent.Is()) { - continue; - } - auto log = logGroup.add_logs(); - std::ostringstream oss; - // set __labels__ - bool hasPrev = false; - for (auto it = metricEvent.TagsBegin(); it != metricEvent.TagsEnd(); ++it) { - if (hasPrev) { - oss << METRIC_LABELS_SEPARATOR; + if (group.mEvents.empty()) { + errorMsg = "empty event group"; + return false; + } + + PipelineEvent::Type eventType = group.mEvents[0]->GetType(); + if (eventType == PipelineEvent::Type::NONE) { + errorMsg = "unsupported event type in event group"; + return false; + } + + bool enableNs = mFlusher->GetContext().GetGlobalConfig().mEnableTimestampNanosecond; + + // caculate serialized logGroup size first, where some critical results can be cached + vector logSZ(group.mEvents.size()); + vector> metricEventContentCache(group.mEvents.size()); + size_t logGroupSZ = 0; + switch (eventType) { + case PipelineEvent::Type::LOG: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& e = group.mEvents[i].Cast(); + size_t contentSZ = 0; + for (const auto& kv : e) { + contentSZ += GetLogContentSize(kv.first.size(), kv.second.size()); } - hasPrev = true; - oss << it->first << METRIC_LABELS_KEY_VALUE_SEPARATOR << it->second; + logGroupSZ += GetLogSize(contentSZ, enableNs && e.GetTimestampNanosecond(), logSZ[i]); } - auto logPtr = log->add_contents(); - logPtr->set_key(METRIC_RESERVED_KEY_LABELS); - logPtr->set_value(oss.str()); - // set time, no need to set nanosecond for metric - log->set_time(metricEvent.GetTimestamp()); - // set __time_nano__ - logPtr = log->add_contents(); - logPtr->set_key(METRIC_RESERVED_KEY_TIME_NANO); - if (metricEvent.GetTimestampNanosecond()) { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) - + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); - } else { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); - } - // set __value__ - if (metricEvent.Is()) { - double value = metricEvent.GetValue()->mValue; - logPtr = log->add_contents(); - logPtr->set_key(METRIC_RESERVED_KEY_VALUE); - logPtr->set_value(std::to_string(value)); + break; + case PipelineEvent::Type::METRIC: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& e = group.mEvents[i].Cast(); + if (e.Is()) { + metricEventContentCache[i].first = to_string(e.GetValue()->mValue); + } else { + LOG_ERROR(sLogger, + ("unexpected error", + "invalid metric event type")("config", mFlusher->GetContext().GetConfigName())); + continue; + } + metricEventContentCache[i].second = GetMetricLabelSize(e); + + size_t contentSZ = 0; + contentSZ += GetLogContentSize(METRIC_RESERVED_KEY_NAME.size(), e.GetName().size()); + contentSZ + += GetLogContentSize(METRIC_RESERVED_KEY_VALUE.size(), metricEventContentCache[i].first.size()); + contentSZ + += GetLogContentSize(METRIC_RESERVED_KEY_TIME_NANO.size(), e.GetTimestampNanosecond() ? 19U : 10U); + contentSZ += GetLogContentSize(METRIC_RESERVED_KEY_LABELS.size(), metricEventContentCache[i].second); + logGroupSZ += GetLogSize(contentSZ, false, logSZ[i]); } - // set __name__ - logPtr = log->add_contents(); - logPtr->set_key(METRIC_RESERVED_KEY_NAME); - logPtr->set_value(metricEvent.GetName().to_string()); + break; + case PipelineEvent::Type::SPAN: + break; + default: + break; + } + // loggroup.category is deprecated, no need to set + for (const auto& tag : group.mTags.mInner) { + if (tag.first == LOG_RESERVED_KEY_TOPIC || tag.first == LOG_RESERVED_KEY_SOURCE + || tag.first == LOG_RESERVED_KEY_MACHINE_UUID) { + logGroupSZ += GetStringSize(tag.second.size()); } else { - errorMsg = "unsupported event type in event group"; - return false; + logGroupSZ += GetLogTagSize(tag.first.size(), tag.second.size()); } } + + if (static_cast(logGroupSZ) > INT32_FLAG(max_send_log_group_size)) { + errorMsg = "log group exceeds size limit\tgroup size: " + ToString(logGroupSZ) + + "\tsize limit: " + ToString(INT32_FLAG(max_send_log_group_size)); + return false; + } + + static LogGroupSerializer serializer; + serializer.Prepare(logGroupSZ); + switch (eventType) { + case PipelineEvent::Type::LOG: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& logEvent = group.mEvents[i].Cast(); + serializer.StartToAddLog(logSZ[i]); + serializer.AddLogTime(logEvent.GetTimestamp()); + for (const auto& kv : logEvent) { + serializer.AddLogContent(kv.first, kv.second); + } + if (enableNs && logEvent.GetTimestampNanosecond()) { + serializer.AddLogTimeNs(logEvent.GetTimestampNanosecond().value()); + } + } + break; + case PipelineEvent::Type::METRIC: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& metricEvent = group.mEvents[i].Cast(); + if (metricEvent.Is()) { + continue; + } + serializer.StartToAddLog(logSZ[i]); + serializer.AddLogTime(metricEvent.GetTimestamp()); + serializer.AddLogContentMetricLabel(metricEvent, metricEventContentCache[i].second); + serializer.AddLogContentMetricTimeNano(metricEvent); + serializer.AddLogContent(METRIC_RESERVED_KEY_VALUE, metricEventContentCache[i].first); + serializer.AddLogContent(METRIC_RESERVED_KEY_NAME, metricEvent.GetName()); + } + break; + case PipelineEvent::Type::SPAN: + break; + default: + break; + } for (const auto& tag : group.mTags.mInner) { if (tag.first == LOG_RESERVED_KEY_TOPIC) { - logGroup.set_topic(tag.second.to_string()); + serializer.AddTopic(tag.second); } else if (tag.first == LOG_RESERVED_KEY_SOURCE) { - logGroup.set_source(tag.second.to_string()); + serializer.AddSource(tag.second); } else if (tag.first == LOG_RESERVED_KEY_MACHINE_UUID) { - logGroup.set_machineuuid(tag.second.to_string()); + serializer.AddMachineUUID(tag.second); } else { - auto logTag = logGroup.add_logtags(); - logTag->set_key(tag.first.to_string()); - logTag->set_value(tag.second.to_string()); + serializer.AddLogTag(tag.first, tag.second); } } - // loggroup.category is deprecated, no need to set - size_t size = logGroup.ByteSizeLong(); - if (static_cast(size) > INT32_FLAG(max_send_log_group_size)) { - errorMsg = "log group exceeds size limit\tgroup size: " + ToString(size) - + "\tsize limit: " + ToString(INT32_FLAG(max_send_log_group_size)); - return false; - } - logGroup.SerializeToString(&res); + res = std::move(serializer.GetResult()); return true; } diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 6ba452f564..9f5b4cd08b 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -486,11 +486,7 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mPluginID, CompressType::LZ4); } -#ifdef __ENTERPRISE__ - mGroupSerializer = make_unique(this); -#else mGroupSerializer = make_unique(this); -#endif mGroupListSerializer = make_unique(this); // MaxSendRate diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 16019ac225..9fb043d9cf 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -30,11 +30,7 @@ #include "pipeline/batch/Batcher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" #include "pipeline/plugin/interface/HttpFlusher.h" -#ifdef __ENTERPRISE__ -#include "pipeline/serializer/EnterpriseSLSSerializer.h" -#else #include "pipeline/serializer/SLSSerializer.h" -#endif #include "protobuf/sls/sls_logs.pb.h" namespace logtail { diff --git a/core/protobuf/sls/LogGroupSerializer.cpp b/core/protobuf/sls/LogGroupSerializer.cpp new file mode 100644 index 0000000000..41c4431133 --- /dev/null +++ b/core/protobuf/sls/LogGroupSerializer.cpp @@ -0,0 +1,259 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "protobuf/sls/LogGroupSerializer.h" + +#include "common/TimeUtil.h" + +using namespace std; + +namespace logtail { + +const string METRIC_RESERVED_KEY_NAME = "__name__"; +const string METRIC_RESERVED_KEY_LABELS = "__labels__"; +const string METRIC_RESERVED_KEY_VALUE = "__value__"; +const string METRIC_RESERVED_KEY_TIME_NANO = "__time_nano__"; + +const string METRIC_LABELS_SEPARATOR = "|"; +const string METRIC_LABELS_KEY_VALUE_SEPARATOR = "#$#"; + +/** + * Return the number of bytes required to store a variable-length unsigned + * 32-bit integer in base-128 varint encoding. + * + * \param v + * Value to encode. + * \return + * Number of bytes required. + */ +static inline size_t uint32_size(uint32_t v) { + if (v < (1UL << 7)) { + return 1; + } else if (v < (1UL << 14)) { + return 2; + } else if (v < (1UL << 21)) { + return 3; + } else if (v < (1UL << 28)) { + return 4; + } else { + return 5; + } +} + +/** + * Pack an unsigned 32-bit integer in base-128 varint encoding and return the + * number of bytes written, which must be 5 or less. + * + * \param value + * Value to encode. + * \param[out] out + * Packed value. + * \return + * Number of bytes written to `out`. + */ +static inline size_t uint32_pack(uint32_t value, string& output) { + unsigned rv = 1; + + if (value >= 0x80) { + output.push_back(value | 0x80); + ++rv; + value >>= 7; + if (value >= 0x80) { + output.push_back(value | 0x80); + ++rv; + value >>= 7; + if (value >= 0x80) { + output.push_back(value | 0x80); + ++rv; + value >>= 7; + if (value >= 0x80) { + output.push_back(value | 0x80); + ++rv; + value >>= 7; + } + } + } + } + /* assert: value<128 */ + output.push_back(value); + return rv; +} + +static inline void fixed32_pack(uint32_t value, string& output) { + for (size_t i = 0; i < 4; ++i) { + output.push_back(value & 0xFF); + value >>= 8; + } +} + +void LogGroupSerializer::Prepare(size_t size) { + mRes.clear(); + mRes.reserve(size); +} + +void LogGroupSerializer::StartToAddLog(size_t size) { + mRes.push_back(0x0A); + uint32_pack(size, mRes); +} + +void LogGroupSerializer::AddLogTime(uint32_t logTime) { + // limit logTime's min value, ensure varint size is 5, which is 1978-07-05 05:24:16 + static uint32_t minLogTime = 1UL << 28; + if (logTime < minLogTime) { + logTime = minLogTime; + } + mRes.push_back(0x08); + uint32_pack(logTime, mRes); +} + +void LogGroupSerializer::AddLogContent(StringView key, StringView value) { + // Contents + mRes.push_back(0x12); + uint32_pack(GetStringSize(key.size()) + GetStringSize(value.size()), mRes); + // Key + mRes.push_back(0x0A); + uint32_pack(key.size(), mRes); + mRes.append(key.data(), key.size()); + // Value + mRes.push_back(0x12); + uint32_pack(value.size(), mRes); + mRes.append(value.data(), value.size()); +} + +void LogGroupSerializer::AddLogTimeNs(uint32_t logTimeNs) { + mRes.push_back(0x25); + fixed32_pack(logTimeNs, mRes); +} + +void LogGroupSerializer::AddTopic(StringView topic) { + mRes.push_back(0x1A); + AddString(topic); +} + +void LogGroupSerializer::AddSource(StringView source) { + mRes.push_back(0x22); + AddString(source); +} + +void LogGroupSerializer::AddMachineUUID(StringView machineUUID) { + mRes.push_back(0x2A); + AddString(machineUUID); +} + +void LogGroupSerializer::AddLogTag(StringView key, StringView value) { + // LogTags + mRes.push_back(0x32); + uint32_pack(GetStringSize(key.size()) + GetStringSize(value.size()), mRes); + // Key + mRes.push_back(0x0A); + uint32_pack(key.size(), mRes); + mRes.append(key.data(), key.size()); + // Value + mRes.push_back(0x12); + uint32_pack(value.size(), mRes); + mRes.append(value.data(), value.size()); +} + +void LogGroupSerializer::AddString(StringView value) { + uint32_pack(value.size(), mRes); + mRes.append(value.data(), value.size()); +} + +void LogGroupSerializer::AddLogContentMetricLabel(const MetricEvent& e, size_t valueSZ) { + // Contents + mRes.push_back(0x12); + uint32_pack(GetStringSize(METRIC_RESERVED_KEY_LABELS.size()) + GetStringSize(valueSZ), mRes); + // Key + mRes.push_back(0x0A); + uint32_pack(METRIC_RESERVED_KEY_LABELS.size(), mRes); + mRes.append(METRIC_RESERVED_KEY_LABELS); + // Value + mRes.push_back(0x12); + uint32_pack(valueSZ, mRes); + bool hasPrev = false; + for (auto it = e.TagsBegin(); it != e.TagsEnd(); ++it) { + if (hasPrev) { + mRes.append(METRIC_LABELS_SEPARATOR); + } + hasPrev = true; + mRes.append(it->first.data(), it->first.size()); + mRes.append(METRIC_LABELS_KEY_VALUE_SEPARATOR); + mRes.append(it->second.data(), it->second.size()); + } +} + +void LogGroupSerializer::AddLogContentMetricTimeNano(const MetricEvent& e) { + size_t valueSZ = e.GetTimestampNanosecond() ? 19U : 10U; + // Contents + mRes.push_back(0x12); + uint32_pack(GetStringSize(METRIC_RESERVED_KEY_TIME_NANO.size()) + GetStringSize(valueSZ), mRes); + // Key + mRes.push_back(0x0A); + uint32_pack(METRIC_RESERVED_KEY_TIME_NANO.size(), mRes); + mRes.append(METRIC_RESERVED_KEY_TIME_NANO); + // Value + mRes.push_back(0x12); + uint32_pack(valueSZ, mRes); + // TODO: avoid copy + mRes.append(to_string(e.GetTimestamp())); + if (e.GetTimestampNanosecond()) { + mRes.append(NumberToDigitString(e.GetTimestampNanosecond().value(), 9)); + } +} + +size_t GetLogContentSize(size_t keySZ, size_t valueSZ) { + size_t res = 0; + res += GetStringSize(keySZ) + GetStringSize(valueSZ); + res += 1 + uint32_size(res); + return res; +} + +size_t GetLogSize(size_t contentSZ, bool hasNs, size_t& logSZ) { + // Contents + size_t res = contentSZ; + // Time, assume varint size is 5 + res += 1 + 5; + // Time_ns + if (hasNs) { + res += 1 + 4; + } + logSZ = res; + // Logs + res += 1 + uint32_size(res); + return res; +} + +size_t GetStringSize(size_t size) { + return 1 + uint32_size(size) + size; +} + +size_t GetLogTagSize(size_t keySZ, size_t valueSZ) { + size_t res = 0; + res += GetStringSize(keySZ) + GetStringSize(valueSZ); + res += 1 + uint32_size(res); + return res; +} + +size_t GetMetricLabelSize(const MetricEvent& e) { + static size_t labelSepSZ = METRIC_LABELS_SEPARATOR.size(); + static size_t keyValSepSZ = METRIC_LABELS_KEY_VALUE_SEPARATOR.size(); + + size_t valueSZ = e.TagsSize() * keyValSepSZ + (e.TagsSize() - 1) * labelSepSZ; + for (auto it = e.TagsBegin(); it != e.TagsEnd(); ++it) { + valueSZ += it->first.size() + it->second.size(); + } + return valueSZ; +} + +} // namespace logtail diff --git a/core/protobuf/sls/LogGroupSerializer.h b/core/protobuf/sls/LogGroupSerializer.h new file mode 100644 index 0000000000..98dbe42f67 --- /dev/null +++ b/core/protobuf/sls/LogGroupSerializer.h @@ -0,0 +1,64 @@ +/* + * Copyright 2023 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "models/StringView.h" +#include "models/MetricEvent.h" + +namespace logtail { + +extern const std::string METRIC_RESERVED_KEY_NAME; +extern const std::string METRIC_RESERVED_KEY_LABELS; +extern const std::string METRIC_RESERVED_KEY_VALUE; +extern const std::string METRIC_RESERVED_KEY_TIME_NANO; + +extern const std::string METRIC_LABELS_SEPARATOR; +extern const std::string METRIC_LABELS_KEY_VALUE_SEPARATOR; + +class LogGroupSerializer { +public: + void Prepare(size_t size); + void StartToAddLog(size_t size); + void AddLogTime(uint32_t logTime); + void AddLogContent(StringView key, StringView value); + void AddLogTimeNs(uint32_t logTimeNs); + void AddTopic(StringView topic); + void AddSource(StringView source); + void AddMachineUUID(StringView machineUUID); + void AddLogTag(StringView key, StringView value); + std::string& GetResult() { return mRes; } + + void AddLogContentMetricLabel(const MetricEvent& e, size_t valueSZ); + void AddLogContentMetricTimeNano(const MetricEvent& e); + +private: + void AddString(StringView value); + + std::string mRes; +}; + +size_t GetLogContentSize(size_t keySZ, size_t valueSZ); +size_t GetLogSize(size_t contentSZ, bool hasNs, size_t& logSZ); +size_t GetStringSize(size_t size); +size_t GetLogTagSize(size_t keySZ, size_t valueSZ); + +size_t GetMetricLabelSize(const MetricEvent& e); + +} // namespace logtail diff --git a/core/unittest/log_pb/CMakeLists.txt b/core/unittest/log_pb/CMakeLists.txt index 6a7ad3019c..e69728ef57 100644 --- a/core/unittest/log_pb/CMakeLists.txt +++ b/core/unittest/log_pb/CMakeLists.txt @@ -15,12 +15,8 @@ cmake_minimum_required(VERSION 3.22) project(log_pb_unittest) -if (ENABLE_ENTERPRISE) - add_executable(log_group_serializer_unittest LogGroupSerializerUnittest.cpp) - target_link_libraries(log_group_serializer_unittest ${UT_BASE_TARGET}) -endif () +add_executable(log_group_serializer_unittest LogGroupSerializerUnittest.cpp) +target_link_libraries(log_group_serializer_unittest ${UT_BASE_TARGET}) include(GoogleTest) -if (ENABLE_ENTERPRISE) - gtest_discover_tests(log_group_serializer_unittest) -endif () +gtest_discover_tests(log_group_serializer_unittest) diff --git a/core/unittest/log_pb/LogGroupSerializerUnittest.cpp b/core/unittest/log_pb/LogGroupSerializerUnittest.cpp new file mode 100644 index 0000000000..262b46ef55 --- /dev/null +++ b/core/unittest/log_pb/LogGroupSerializerUnittest.cpp @@ -0,0 +1,103 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "protobuf/sls/LogGroupSerializer.h" +#include "protobuf/sls/sls_logs.pb.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class LogGroupLiteUnittest : public ::testing::Test { +public: + void TestSerialize(); +}; + +void LogGroupLiteUnittest::TestSerialize() { + size_t groupSZ = 0; + vector logSZ(2); + { + auto contentSZ = 0; + contentSZ += GetLogContentSize(strlen("key_1"), strlen("value_1")); + contentSZ += GetLogContentSize(strlen("key_2"), strlen("value_2")); + groupSZ += GetLogSize(contentSZ, true, logSZ[0]); + } + { + auto contentSZ = 0; + contentSZ += GetLogContentSize(strlen("key_3"), strlen("value_3")); + contentSZ += GetLogContentSize(strlen("key_4"), strlen("value_4")); + groupSZ += GetLogSize(contentSZ, false, logSZ[1]); + } + groupSZ += GetStringSize(strlen("topic")); + groupSZ += GetStringSize(strlen("source")); + groupSZ += GetStringSize(strlen("machine_uuid")); + groupSZ += GetLogTagSize(strlen("key_5"), strlen("value_5")); + groupSZ += GetLogTagSize(strlen("key_6"), strlen("value_6")); + + LogGroupLite logGroup; + logGroup.PrepareToSerialize(groupSZ); + logGroup.StartToAddLog(logSZ[0]); + logGroup.AddLogTime(1234567890); + logGroup.AddLogContent("key_1", "value_1"); + logGroup.AddLogContent("key_2", "value_2"); + logGroup.AddLogTimeNs(135792468); + logGroup.StartToAddLog(logSZ[1]); + logGroup.AddLogTime(123456789); + logGroup.AddLogContent("key_3", "value_3"); + logGroup.AddLogContent("key_4", "value_4"); + logGroup.AddTopic("topic"); + logGroup.AddSource("source"); + logGroup.AddMachineUUID("machine_uuid"); + logGroup.AddLogTag("key_5", "value_5"); + logGroup.AddLogTag("key_6", "value_6"); + APSARA_TEST_EQUAL(groupSZ, logGroup.GetResult().size()); + + sls_logs::LogGroup logGroupPb; + APSARA_TEST_TRUE(logGroupPb.ParseFromString(logGroup.GetResult())); + APSARA_TEST_EQUAL(2L, logGroupPb.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroupPb.logs(0).time()); + APSARA_TEST_TRUE(logGroupPb.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(135792468U, logGroupPb.logs(0).time_ns()); + APSARA_TEST_EQUAL(2L, logGroupPb.logs(0).contents_size()); + APSARA_TEST_EQUAL("key_1", logGroupPb.logs(0).contents(0).key()); + APSARA_TEST_EQUAL("value_1", logGroupPb.logs(0).contents(0).value()); + APSARA_TEST_EQUAL("key_2", logGroupPb.logs(0).contents(1).key()); + APSARA_TEST_EQUAL("value_2", logGroupPb.logs(0).contents(1).value()); + APSARA_TEST_EQUAL(1L << 28, logGroupPb.logs(1).time()); + APSARA_TEST_FALSE(logGroupPb.logs(1).has_time_ns()); + APSARA_TEST_EQUAL(2L, logGroupPb.logs(1).contents_size()); + APSARA_TEST_EQUAL("key_3", logGroupPb.logs(1).contents(0).key()); + APSARA_TEST_EQUAL("value_3", logGroupPb.logs(1).contents(0).value()); + APSARA_TEST_EQUAL("key_4", logGroupPb.logs(1).contents(1).key()); + APSARA_TEST_EQUAL("value_4", logGroupPb.logs(1).contents(1).value()); + APSARA_TEST_TRUE(logGroupPb.has_topic()); + APSARA_TEST_EQUAL("topic", logGroupPb.topic()); + APSARA_TEST_TRUE(logGroupPb.has_source()); + APSARA_TEST_EQUAL("source", logGroupPb.source()); + APSARA_TEST_TRUE(logGroupPb.has_machineuuid()); + APSARA_TEST_EQUAL("machine_uuid", logGroupPb.machineuuid()); + APSARA_TEST_FALSE(logGroupPb.has_category()); + APSARA_TEST_EQUAL(2L, logGroupPb.logtags_size()); + APSARA_TEST_EQUAL("key_5", logGroupPb.logtags(0).key()); + APSARA_TEST_EQUAL("value_5", logGroupPb.logtags(0).value()); + APSARA_TEST_EQUAL("key_6", logGroupPb.logtags(1).key()); + APSARA_TEST_EQUAL("value_6", logGroupPb.logtags(1).value()); +} + +UNIT_TEST_CASE(LogGroupLiteUnittest, TestSerialize) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/serializer/CMakeLists.txt b/core/unittest/serializer/CMakeLists.txt index 6fb442365f..b471e3b81a 100644 --- a/core/unittest/serializer/CMakeLists.txt +++ b/core/unittest/serializer/CMakeLists.txt @@ -18,18 +18,9 @@ project(serializer_unittest) add_executable(serializer_unittest SerializerUnittest.cpp) target_link_libraries(serializer_unittest ${UT_BASE_TARGET}) -if (ENABLE_ENTERPRISE) - add_executable(enterprise_sls_serializer_unittest EnterpriseSLSSerializerUnittest.cpp) - target_link_libraries(enterprise_sls_serializer_unittest ${UT_BASE_TARGET}) -else () - add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) - target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) -endif () +add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) +target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) include(GoogleTest) gtest_discover_tests(serializer_unittest) -if (ENABLE_ENTERPRISE) - gtest_discover_tests(enterprise_sls_serializer_unittest) -else () - gtest_discover_tests(sls_serializer_unittest) -endif () +gtest_discover_tests(sls_serializer_unittest) From 1ef689009b902eb4aa24d6ea668afa436222c76b Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 1 Nov 2024 12:41:04 +0000 Subject: [PATCH 07/12] polish --- core/pipeline/serializer/SLSSerializer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index e43ec49104..106571b1ce 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/serializer/EnterpriseSLSSerializer.h" +#include "pipeline/serializer/SLSSerializer.h" #include "common/Flags.h" #include "common/compression/CompressType.h" From 0c17a66978741121d47fa377aacab5165c173d84 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 1 Nov 2024 13:25:00 +0000 Subject: [PATCH 08/12] polish --- core/unittest/log_pb/LogGroupSerializerUnittest.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/unittest/log_pb/LogGroupSerializerUnittest.cpp b/core/unittest/log_pb/LogGroupSerializerUnittest.cpp index 262b46ef55..d202c70558 100644 --- a/core/unittest/log_pb/LogGroupSerializerUnittest.cpp +++ b/core/unittest/log_pb/LogGroupSerializerUnittest.cpp @@ -20,12 +20,12 @@ using namespace std; namespace logtail { -class LogGroupLiteUnittest : public ::testing::Test { +class LogGroupSerializerUnittest : public ::testing::Test { public: void TestSerialize(); }; -void LogGroupLiteUnittest::TestSerialize() { +void LogGroupSerializerUnittest::TestSerialize() { size_t groupSZ = 0; vector logSZ(2); { @@ -46,8 +46,8 @@ void LogGroupLiteUnittest::TestSerialize() { groupSZ += GetLogTagSize(strlen("key_5"), strlen("value_5")); groupSZ += GetLogTagSize(strlen("key_6"), strlen("value_6")); - LogGroupLite logGroup; - logGroup.PrepareToSerialize(groupSZ); + LogGroupSerializer logGroup; + logGroup.Prepare(groupSZ); logGroup.StartToAddLog(logSZ[0]); logGroup.AddLogTime(1234567890); logGroup.AddLogContent("key_1", "value_1"); @@ -64,7 +64,7 @@ void LogGroupLiteUnittest::TestSerialize() { logGroup.AddLogTag("key_6", "value_6"); APSARA_TEST_EQUAL(groupSZ, logGroup.GetResult().size()); - sls_logs::LogGroup logGroupPb; + sls_logs::LogGroup logGroupPb; APSARA_TEST_TRUE(logGroupPb.ParseFromString(logGroup.GetResult())); APSARA_TEST_EQUAL(2L, logGroupPb.logs_size()); APSARA_TEST_EQUAL(1234567890U, logGroupPb.logs(0).time()); @@ -96,7 +96,7 @@ void LogGroupLiteUnittest::TestSerialize() { APSARA_TEST_EQUAL("value_6", logGroupPb.logtags(1).value()); } -UNIT_TEST_CASE(LogGroupLiteUnittest, TestSerialize) +UNIT_TEST_CASE(LogGroupSerializerUnittest, TestSerialize) } // namespace logtail From b443b89e3247ba379ed6f578c31166dd483a3900 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 4 Nov 2024 07:07:40 +0000 Subject: [PATCH 09/12] polish --- core/pipeline/serializer/SLSSerializer.cpp | 10 + core/protobuf/sls/LogGroupSerializer.cpp | 13 + core/protobuf/sls/LogGroupSerializer.h | 1 + .../serializer/SLSSerializerUnittest.cpp | 314 +++++++++++------- 4 files changed, 211 insertions(+), 127 deletions(-) diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 106571b1ce..f448e32a87 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -58,6 +58,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri PipelineEvent::Type eventType = group.mEvents[0]->GetType(); if (eventType == PipelineEvent::Type::NONE) { + // should not happen errorMsg = "unsupported event type in event group"; return false; } @@ -72,6 +73,9 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri case PipelineEvent::Type::LOG: for (size_t i = 0; i < group.mEvents.size(); ++i) { const auto& e = group.mEvents[i].Cast(); + if (e.Empty()) { + continue; + } size_t contentSZ = 0; for (const auto& kv : e) { contentSZ += GetLogContentSize(kv.first.size(), kv.second.size()); @@ -85,6 +89,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri if (e.Is()) { metricEventContentCache[i].first = to_string(e.GetValue()->mValue); } else { + // should not happen LOG_ERROR(sLogger, ("unexpected error", "invalid metric event type")("config", mFlusher->GetContext().GetConfigName())); @@ -107,6 +112,11 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri default: break; } + if (logGroupSZ == 0) { + errorMsg = "all empty log"; + return false; + } + // loggroup.category is deprecated, no need to set for (const auto& tag : group.mTags.mInner) { if (tag.first == LOG_RESERVED_KEY_TOPIC || tag.first == LOG_RESERVED_KEY_SOURCE diff --git a/core/protobuf/sls/LogGroupSerializer.cpp b/core/protobuf/sls/LogGroupSerializer.cpp index 41c4431133..7e94189092 100644 --- a/core/protobuf/sls/LogGroupSerializer.cpp +++ b/core/protobuf/sls/LogGroupSerializer.cpp @@ -103,6 +103,7 @@ void LogGroupSerializer::Prepare(size_t size) { } void LogGroupSerializer::StartToAddLog(size_t size) { + // field = 1, wire_type = 2 mRes.push_back(0x0A); uint32_pack(size, mRes); } @@ -113,53 +114,64 @@ void LogGroupSerializer::AddLogTime(uint32_t logTime) { if (logTime < minLogTime) { logTime = minLogTime; } + // field = 1, wire_type = 0 mRes.push_back(0x08); uint32_pack(logTime, mRes); } void LogGroupSerializer::AddLogContent(StringView key, StringView value) { // Contents + // field = 2, wire_type = 2 mRes.push_back(0x12); uint32_pack(GetStringSize(key.size()) + GetStringSize(value.size()), mRes); // Key + // field = 1, wire_type = 2 mRes.push_back(0x0A); uint32_pack(key.size(), mRes); mRes.append(key.data(), key.size()); // Value + // field = 2, wire_type = 2 mRes.push_back(0x12); uint32_pack(value.size(), mRes); mRes.append(value.data(), value.size()); } void LogGroupSerializer::AddLogTimeNs(uint32_t logTimeNs) { + // field = 4, wire_type = 5 mRes.push_back(0x25); fixed32_pack(logTimeNs, mRes); } void LogGroupSerializer::AddTopic(StringView topic) { + // field = 3, wire_type = 2 mRes.push_back(0x1A); AddString(topic); } void LogGroupSerializer::AddSource(StringView source) { + // field = 4, wire_type = 2 mRes.push_back(0x22); AddString(source); } void LogGroupSerializer::AddMachineUUID(StringView machineUUID) { + // field = 5, wire_type = 2 mRes.push_back(0x2A); AddString(machineUUID); } void LogGroupSerializer::AddLogTag(StringView key, StringView value) { // LogTags + // field = 6, wire_type = 2 mRes.push_back(0x32); uint32_pack(GetStringSize(key.size()) + GetStringSize(value.size()), mRes); // Key + // field = 1, wire_type = 2 mRes.push_back(0x0A); uint32_pack(key.size(), mRes); mRes.append(key.data(), key.size()); // Value + // field = 2, wire_type = 2 mRes.push_back(0x12); uint32_pack(value.size(), mRes); mRes.append(value.data(), value.size()); @@ -226,6 +238,7 @@ size_t GetLogSize(size_t contentSZ, bool hasNs, size_t& logSZ) { res += 1 + 5; // Time_ns if (hasNs) { + // fixed32 size is always 4 res += 1 + 4; } logSZ = res; diff --git a/core/protobuf/sls/LogGroupSerializer.h b/core/protobuf/sls/LogGroupSerializer.h index 98dbe42f67..394bbb9a6d 100644 --- a/core/protobuf/sls/LogGroupSerializer.h +++ b/core/protobuf/sls/LogGroupSerializer.h @@ -32,6 +32,7 @@ extern const std::string METRIC_RESERVED_KEY_TIME_NANO; extern const std::string METRIC_LABELS_SEPARATOR; extern const std::string METRIC_LABELS_KEY_VALUE_SEPARATOR; +// see for detail: https://protobuf.dev/programming-guides/encoding/ class LogGroupSerializer { public: void Prepare(size_t size); diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index caa379e08f..f9d23561d9 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "plugin/flusher/sls/FlusherSLS.h" #include "pipeline/serializer/SLSSerializer.h" +#include "plugin/flusher/sls/FlusherSLS.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(max_send_log_group_size); @@ -37,8 +37,9 @@ class SLSSerializerUnittest : public ::testing::Test { } private: - BatchedEvents CreateBatchedEvents(bool enableNanosecond); - BatchedEvents CreateBatchedMetricEvents(bool enableNanosecond, uint32_t nanoTimestamp, bool emptyValue); + BatchedEvents CreateBatchedLogEvents(bool enableNanosecond, bool emptyContent); + BatchedEvents + CreateBatchedMetricEvents(bool enableNanosecond, uint32_t nanoTimestamp, bool emptyValue, bool onlyOneTag); static unique_ptr sFlusher; @@ -50,139 +51,191 @@ unique_ptr SLSSerializerUnittest::sFlusher; void SLSSerializerUnittest::TestSerializeEventGroup() { SLSEventGroupSerializer serializer(sFlusher.get()); { - // nano second disabled, and set - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); - APSARA_TEST_STREQ("key", logGroup.logs(0).contents(0).key().c_str()); - APSARA_TEST_STREQ("value", logGroup.logs(0).contents(0).value().c_str()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); + // log + { + // nano second disabled, and set + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); + APSARA_TEST_STREQ("key", logGroup.logs(0).contents(0).key().c_str()); + APSARA_TEST_STREQ("value", logGroup.logs(0).contents(0).value().c_str()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); + } + { + // nano second enabled, and set + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(true, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_EQUAL(1U, logGroup.logs(0).time_ns()); + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; + } + { + // nano second enabled, not set + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; + } + { + // empty log content + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, true), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_FALSE(logGroup.ParseFromString(res)); + } } { - // nano second enabled, and set - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_EQUAL(1U, logGroup.logs(0).time_ns()); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; + // metric + { + // only 1 tag + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false, true), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); + } + { + // nano second disabled + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); + } + { + // nano second enabled, less than 9 digits + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890000000001"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); + } + { + // nano second enabled, exactly 9 digits + string res, errorMsg; + APSARA_TEST_TRUE( + serializer.DoSerialize(CreateBatchedMetricEvents(true, 999999999, false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890999999999"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); + } + { + // empty metric value + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_FALSE(logGroup.ParseFromString(res)); + } } { - // nano second enabled, not set - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; + // span } { // log group exceed size limit INT32_FLAG(max_send_log_group_size) = 0; string res, errorMsg; - APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedLogEvents(true, false), res, errorMsg)); INT32_FLAG(max_send_log_group_size) = 10 * 1024 * 1024; } { - // metric event + // empty log group + PipelineEventGroup group(make_shared()); + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - } - { - // metric event with EnableTimestampNanosecond - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890000000001"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; - } - { - // metric event with EnableTimestampNanosecond - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890999999999"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; - } - - { - // metric event with EnableTimestampNanosecond - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(0, logGroup.logs_size()); + APSARA_TEST_FALSE(serializer.DoSerialize(std::move(batch), res, errorMsg)); } } @@ -202,7 +255,7 @@ void SLSSerializerUnittest::TestSerializeEventGroupList() { } -BatchedEvents SLSSerializerUnittest::CreateBatchedEvents(bool enableNanosecond) { +BatchedEvents SLSSerializerUnittest::CreateBatchedLogEvents(bool enableNanosecond, bool emptyContent) { PipelineEventGroup group(make_shared()); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); @@ -212,7 +265,9 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedEvents(bool enableNanosecond) group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); LogEvent* e = group.AddLogEvent(); - e->SetContent(string("key"), string("value")); + if (!emptyContent) { + e->SetContent(string("key"), string("value")); + } if (enableNanosecond) { e->SetTimestamp(1234567890, 1); } else { @@ -227,7 +282,10 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedEvents(bool enableNanosecond) } -BatchedEvents SLSSerializerUnittest::CreateBatchedMetricEvents(bool enableNanosecond, uint32_t nanoTimestamp, bool emptyValue) { +BatchedEvents SLSSerializerUnittest::CreateBatchedMetricEvents(bool enableNanosecond, + uint32_t nanoTimestamp, + bool emptyValue, + bool onlyOneTag) { PipelineEventGroup group(make_shared()); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); @@ -239,7 +297,9 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedMetricEvents(bool enableNanose group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); MetricEvent* e = group.AddMetricEvent(); e->SetTag(string("key1"), string("value1")); - e->SetTag(string("key2"), string("value2")); + if (!onlyOneTag) { + e->SetTag(string("key2"), string("value2")); + } if (enableNanosecond) { e->SetTimestamp(1234567890, nanoTimestamp); } else { From 26c38bd8faeb7ec9ec394dba9e4fa2878dcda4aa Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 4 Nov 2024 07:12:02 +0000 Subject: [PATCH 10/12] polish --- core/pipeline/serializer/SLSSerializer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index f448e32a87..b9161bd2c6 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -133,7 +133,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri return false; } - static LogGroupSerializer serializer; + thread_local LogGroupSerializer serializer; serializer.Prepare(logGroupSZ); switch (eventType) { case PipelineEvent::Type::LOG: From a6362ce2544e9e941d7e2f88032426cda3f0ec0e Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 4 Nov 2024 07:50:20 +0000 Subject: [PATCH 11/12] polish --- core/pipeline/serializer/SLSSerializer.cpp | 2 +- core/unittest/serializer/SLSSerializerUnittest.cpp | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index b9161bd2c6..3f48dc961a 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -113,7 +113,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri break; } if (logGroupSZ == 0) { - errorMsg = "all empty log"; + errorMsg = "all empty logs"; return false; } diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index f9d23561d9..837e48ff8b 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -96,9 +96,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { { // empty log content string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, true), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_FALSE(logGroup.ParseFromString(res)); + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedLogEvents(false, true), res, errorMsg)); } } { @@ -211,9 +209,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { { // empty metric value string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_FALSE(logGroup.ParseFromString(res)); + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true, false), res, errorMsg)); } } { From 319030a375692984669c8d87f7d3b75312a3f5a6 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 4 Nov 2024 11:28:52 +0000 Subject: [PATCH 12/12] polish --- .github/workflows/build-core-ut.yaml | 2 +- docker/Dockerfile_coverage | 2 +- docs/cn/developer-guide/test/unit-test.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-core-ut.yaml b/.github/workflows/build-core-ut.yaml index c0db279b1f..8123a7f672 100644 --- a/.github/workflows/build-core-ut.yaml +++ b/.github/workflows/build-core-ut.yaml @@ -82,7 +82,7 @@ jobs: run: make unittest_core - name: Unit Test Coverage - run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*sdk.*\" -e \".*logger.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*protobuf.*\" -e \".*runner.*\"" + run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*\.pb\.cc\" -e \".*\.pb\.h\" -e \".*unittest.*\" -e \".*sdk.*\" -e \".*logger.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*runner.*\"" - name: Setup Python3.10 uses: actions/setup-python@v5 diff --git a/docker/Dockerfile_coverage b/docker/Dockerfile_coverage index b57dbbc644..97c2af78a4 100644 --- a/docker/Dockerfile_coverage +++ b/docker/Dockerfile_coverage @@ -30,4 +30,4 @@ RUN python3 -m pip install --upgrade pip RUN cp /usr/local/python3/bin/pip3 /usr/bin/pip3 && pip3 install gcovr==7.0 RUN cp /usr/local/python3/bin/gcovr /usr/bin/gcovr -CMD ["bash", "-c", "gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*sdk.*\" -e \".*logger.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*protobuf.*\" -e \".*runner.*\""] +CMD ["bash", "-c", "gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*\.pb\.cc\" -e \".*\.pb\.h\" -e \".*unittest.*\" -e \".*sdk.*\" -e \".*logger.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*runner.*\""] diff --git a/docs/cn/developer-guide/test/unit-test.md b/docs/cn/developer-guide/test/unit-test.md index a660eb9799..ff01bb2e33 100644 --- a/docs/cn/developer-guide/test/unit-test.md +++ b/docs/cn/developer-guide/test/unit-test.md @@ -19,7 +19,7 @@ cmake -DBUILD_LOGTAIL_UT=ON <其他编译参数> .. ```shell mkdir -p coverage-report # 生成详细的报告 -gcovr -r ./core --txt coverage-report/index.txt --html-details --html coverage-report/index.html -e ".*sdk.*" -e ".*observer.*" -e ".*protobuf.*" -e ".*unittest.*" -e ".*config_server.*" -e ".*fuse.*" -e ".*go_pipeline.*" +gcovr -r ./core --txt coverage-report/index.txt --html-details --html coverage-report/index.html -e ".*\.pb\.cc" -e ".*\.pb\.h" -e ".*unittest.*" -e ".*config_server.*" -e ".*go_pipeline.*" -e ".*sdk.*" # 生成本次commit diff的报告 python3 tools/coverage-diff/main.py coverage-report/index.txt ```