From 1293212b264e67d91734c642e5ec454d870fed8b Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 05:11:53 +0000 Subject: [PATCH] fix --- core/reader/LogFileReader.cpp | 73 +++++++++++++++---- core/reader/LogFileReader.h | 8 +- .../RemoveLastIncompleteLogUnittest.cpp | 57 +++++++-------- 3 files changed, 93 insertions(+), 45 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index 92e8361473..0855316b6a 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2050,6 +2050,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll content.data.size(), *mMultilineConfig.first->GetEndPatternReg(), exception)) { + rollbackLineFeedCount += content.forceRollbackLineFeedCount; // Ensure the end line is complete if (buffer[content.lineEnd] == '\n') { return content.lineEnd + 1; @@ -2061,10 +2062,12 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll *mMultilineConfig.first->GetStartPatternReg(), exception)) { // start + continue, start + rollbackLineFeedCount += content.forceRollbackLineFeedCount; rollbackLineFeedCount += content.rollbackLineFeedCount; // Keep all the buffer if rollback all return content.lineBegin; } + rollbackLineFeedCount += content.forceRollbackLineFeedCount; rollbackLineFeedCount += content.rollbackLineFeedCount; endPs = content.lineBegin - 1; } @@ -2280,10 +2283,20 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex != 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } for (int32_t begin = end; begin > 0; --begin) { @@ -2292,14 +2305,16 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer, .lineBegin = begin, .lineEnd = end, .rollbackLineFeedCount = 1, - .fullLine = true}; + .fullLine = true, + .forceRollbackLineFeedCount = 0}; } } return {.data = StringView(buffer.data(), end), .lineBegin = 0, .lineEnd = end, .rollbackLineFeedCount = 1, - .fullLine = true}; + .fullLine = true, + .forceRollbackLineFeedCount = 0}; } LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, @@ -2308,11 +2323,21 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex == 0) { // 异常情况, DockerJsonFileParse不允许在最后一个解析器 - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } size_t nextProtocolFunctionIndex = protocolFunctionIndex - 1; @@ -2326,7 +2351,12 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + if (line.fullLine) { + line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + } else { + line.forceRollbackLineFeedCount += line.rollbackLineFeedCount; + } finalLine = std::move(line); if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { @@ -2334,7 +2364,8 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, - .fullLine = false}; + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } end = finalLine.lineBegin - 1; } @@ -2385,11 +2416,21 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex == 0) { - // 异常情况, DockerJsonFileParse不允许在最后一个解析器 - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + // 异常情况, ContainerdTextParser不允许在最后一个解析器 + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } LineInfo finalLine; finalLine.fullLine = false; @@ -2405,7 +2446,12 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + if (line.fullLine) { + line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + } else { + line.forceRollbackLineFeedCount += line.rollbackLineFeedCount; + } finalLine = std::move(line); mergeLines(finalLine, finalLine, true); if (!finalLine.fullLine) { @@ -2414,7 +2460,8 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, - .fullLine = false}; + .fullLine = false, + .forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount}; } end = finalLine.lineBegin - 1; } diff --git a/core/reader/LogFileReader.h b/core/reader/LogFileReader.h index 0a75f0856b..4eb6309ae3 100644 --- a/core/reader/LogFileReader.h +++ b/core/reader/LogFileReader.h @@ -34,7 +34,6 @@ #include "event/Event.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/FileServer.h" -#include "file_server/FileServer.h" #include "file_server/MultilineOptions.h" #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" @@ -57,16 +56,19 @@ struct LineInfo { int32_t lineEnd; int32_t rollbackLineFeedCount; bool fullLine; + int32_t forceRollbackLineFeedCount; LineInfo(StringView data = StringView(), int32_t lineBegin = 0, int32_t lineEnd = 0, int32_t rollbackLineFeedCount = 0, - bool fullLine = false) + bool fullLine = false, + int32_t forceRollbackLineFeedCount = 0) : data(data), lineBegin(lineBegin), lineEnd(lineEnd), rollbackLineFeedCount(rollbackLineFeedCount), - fullLine(fullLine) {} + fullLine(fullLine), + forceRollbackLineFeedCount(forceRollbackLineFeedCount) {} }; class BaseLineParse { diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 92cca93dd6..c52966c7c5 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -448,14 +448,14 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + std::string expectMatch = "\n\n" + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } } } @@ -487,7 +487,7 @@ void GetLastLineUnittest::TestGetLastLineEmpty() { LogFileReader logFileReader( "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(nullptr, &ctx)); auto lastLine = logFileReader.GetLastLine(const_cast(testLog.data()), testLog.size()); - APSARA_TEST_EQUAL_FATAL(0, lastLine.size()); + APSARA_TEST_EQUAL_FATAL(0, int(lastLine.size())); APSARA_TEST_EQUAL_FATAL("", std::string(lastLine.data(), lastLine.size())); APSARA_TEST_EQUAL_FATAL(testLog.data(), lastLine.data()); } @@ -547,7 +547,6 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - // logFileReader.mDiscardUnmatch = true; { // case: end with end std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; @@ -558,9 +557,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } { // case: end with unmatch std::string expectMatch @@ -572,9 +571,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); } { // case: all unmatch { @@ -586,38 +585,38 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); } { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } } { std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART - + LOG_UNMATCH + "\n"; + std::string testLog + = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } { std::string expectMatch @@ -630,9 +629,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } }