From 5882b8f0e381aa4ff81f2c79b0672b6a5a6da90b Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 02:25:20 +0000 Subject: [PATCH 01/12] fix --- core/reader/LogFileReader.cpp | 58 ++++---- .../reader/GetLastLineDataUnittest.cpp | 40 ++++- .../RemoveLastIncompleteLogUnittest.cpp | 139 ++++++++++++++++++ 3 files changed, 211 insertions(+), 26 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index b20bbf71fb..f777c0525f 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -50,7 +50,6 @@ #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" #include "monitor/MetricConstants.h" -#include "monitor/MetricConstants.h" #include "processor/inner/ProcessorParseContainerLogNative.h" #include "rapidjson/document.h" #include "reader/JsonLogFileReader.h" @@ -2073,13 +2072,17 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll // Single line rollback or all unmatch rollback rollbackLineFeedCount = 0; if (buffer[size - 1] == '\n') { + if (mReaderConfig.first->mInputType == FileReaderOptions::InputType::InputFile) { + // 如果是文件采集，直接返回，节约性能 + return size; + } endPs = size - 1; } else { endPs = size; } LineInfo content = NewGetLastLine(StringView(buffer, size), endPs, true); // 最后一行是完整行,且以 \n 结尾 - if (content.fullLine && buffer[endPs] == '\n') { + if (content.fullLine && buffer[size - 1] == '\n' && content.lineEnd == endPs) { return size; } content = NewGetLastLine(StringView(buffer, size), endPs, false); @@ -2288,7 +2291,7 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer, } for (int32_t begin = end; begin > 0; --begin) { - if (begin == 0 || buffer[begin - 1] == '\n') { + if (buffer[begin - 1] == '\n') { return {.data = StringView(buffer.data() + begin, end - begin), .lineBegin = begin, .lineEnd = end, @@ -2321,24 +2324,21 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, while (!finalLine.fullLine) { LineInfo rawLine = (*lineParsers)[nextProtocolFunctionIndex]->NewGetLastLine( buffer, end, nextProtocolFunctionIndex, needSingleLine, lineParsers); - if (rawLine.data.back() == '\n') { + if (rawLine.data.size() > 0 && rawLine.data.back() == '\n') { rawLine.data = StringView(rawLine.data.data(), rawLine.data.size() - 1); } LineInfo line; parseLine(rawLine, line); - finalLine.data = line.data; - finalLine.fullLine = line.fullLine; - finalLine.lineBegin = line.lineBegin; - finalLine.rollbackLineFeedCount += line.rollbackLineFeedCount; - finalLine.dataRaw = line.dataRaw; - if (finalLine.lineEnd == 0) { - finalLine.lineEnd = line.lineEnd; - } + line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + finalLine = std::move(line); if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { - finalLine.data = StringView(); - return finalLine; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, + .fullLine = false}; } end = finalLine.lineBegin - 1; } @@ -2349,7 +2349,9 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, bool DockerJsonFileParser::parseLine(LineInfo rawLine, LineInfo& paseLine) { paseLine = rawLine; paseLine.fullLine = false; - + if (rawLine.data.size() == 0) { + return false; + } rapidjson::Document doc; doc.Parse(rawLine.data.data(), rawLine.data.size()); @@ -2395,13 +2397,13 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, } LineInfo finalLine; finalLine.fullLine = false; - // 跳过最后的连续P size_t nextProtocolFunctionIndex = protocolFunctionIndex - 1; + // 跳过最后的连续P while (!finalLine.fullLine) { LineInfo rawLine = (*lineParsers)[nextProtocolFunctionIndex]->NewGetLastLine( buffer, end, nextProtocolFunctionIndex, needSingleLine, lineParsers); - if (rawLine.data.back() == '\n') { + if (rawLine.data.size() > 0 && rawLine.data.back() == '\n') { rawLine.data = StringView(rawLine.data.data(), rawLine.data.size() - 1); } @@ -2412,14 +2414,15 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, finalLine.fullLine = line.fullLine; finalLine.lineBegin = line.lineBegin; finalLine.rollbackLineFeedCount += line.rollbackLineFeedCount; + finalLine.lineEnd = line.lineEnd; mergeLines(finalLine, finalLine, true); - if (finalLine.lineEnd == 0) { - finalLine.lineEnd = line.lineEnd; - } if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { - finalLine.data = StringView(); - return finalLine; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, + .fullLine = false}; } end = finalLine.lineBegin - 1; } @@ -2442,7 +2445,7 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo previousLine; LineInfo rawLine = (*lineParsers)[nextProtocolFunctionIndex]->NewGetLastLine( buffer, finalLine.lineBegin - 1, nextProtocolFunctionIndex, needSingleLine, lineParsers); - if (rawLine.data.back() == '\n') { + if (rawLine.data.size() > 0 && rawLine.data.back() == '\n') { rawLine.data = StringView(rawLine.data.data(), rawLine.data.size() - 1); } @@ -2479,9 +2482,10 @@ void ContainerdTextParser::parseLine(LineInfo rawLine, LineInfo& paseLine) { const char* lineEnd = rawLine.data.data() + rawLine.data.size(); paseLine = rawLine; paseLine.fullLine = true; - + if (rawLine.data.size() == 0) { + return; + } // 寻找第一个分隔符位置 time - StringView timeValue; const char* pch1 = std::find(rawLine.data.data(), lineEnd, ProcessorParseContainerLogNative::CONTAINERD_DELIMITER); if (pch1 == lineEnd) { return; @@ -2497,6 +2501,10 @@ void ContainerdTextParser::parseLine(LineInfo rawLine, LineInfo& paseLine) { return; } // 如果既不以 P 开头,也不以 F 开头 + if (pch2 + 1 >= lineEnd) { + paseLine.data = StringView(pch2 + 1, lineEnd - pch2 - 1); + return; + } if (*(pch2 + 1) != ProcessorParseContainerLogNative::CONTAINERD_PART_TAG && *(pch2 + 1) != ProcessorParseContainerLogNative::CONTAINERD_FULL_TAG) { paseLine.data = StringView(pch2 + 1, lineEnd - pch2 - 1); diff --git a/core/unittest/reader/GetLastLineDataUnittest.cpp b/core/unittest/reader/GetLastLineDataUnittest.cpp index 8ed6cd2f33..a90ab57a8c 100644 --- a/core/unittest/reader/GetLastLineDataUnittest.cpp +++ b/core/unittest/reader/GetLastLineDataUnittest.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include "common/FileSystemUtil.h" -#include "reader/LogFileReader.h" #include "common/memory/SourceBuffer.h" +#include "reader/LogFileReader.h" #include "unittest/Unittest.h" namespace logtail { @@ -516,6 +516,44 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { + { + std::string testLog = "\n2024-01-05T23:28:06.818486411+08:00 stdout P 123123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.lineEnd); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + { + std::string testLog = "\n2024-01-05T23:28:06.818486411+08:00 stdout F 123123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123123", line.data.to_string()); + APSARA_TEST_EQUAL(1, line.lineBegin); + APSARA_TEST_EQUAL(endPs, line.lineEnd); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + } // 异常情况+有回车 { // case: PartLogFlag存在，第三个空格存在但空格后无内容 diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 19e20463e2..f63183c515 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -446,6 +446,16 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + } } class GetLastLineUnittest : public ::testing::Test { @@ -480,6 +490,135 @@ void GetLastLineUnittest::TestGetLastLineEmpty() { APSARA_TEST_EQUAL_FATAL(testLog.data(), lastLine.data()); } +class ContainerdTextRemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { +public: + std::shared_ptr GetLogFileReader(const Json::Value& config) { + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + auto logFileReader = std::make_shared( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader->GetParser(LogFileReader::BUFFER_SIZE); + logFileReader->mLineParsers.emplace_back(baseLineParsePtr); + return logFileReader; + } + void TestRemoveLastIncompleteLogWithBeginContinue(); + void TestRemoveLastIncompleteLogWithBeginEnd(); + void TestRemoveLastIncompleteLogWithBegin(); + void TestRemoveLastIncompleteLogWithContinueEnd(); + void TestRemoveLastIncompleteLogWithEnd(); + void SetUp() override { readerOpts.mInputType = FileReaderOptions::InputType::InputContainerStdio; } + +private: + FileReaderOptions readerOpts; + PipelineContext ctx; + const std::string LOG_PART = "2021-08-25T07:00:00.000000000Z stdout P "; + const std::string LOG_FULL = "2021-08-25T07:00:00.000000000Z stdout F "; + const std::string LOG_FULL_NOT_FOUND = "2021-08-25T07:00:00.000000000Z stdout "; + const std::string LOG_ERROR = "2021-08-25T07:00:00.000000000Z stdout"; + + const std::string LOG_BEGIN_STRING = "Exception in thread \"main\" java.lang.NullPointerException"; + const std::string LOG_BEGIN_REGEX = R"(Exception.*)"; + + const std::string LOG_CONTINUE_STRING = " at com.example.myproject.Book.getTitle(Book.java:16)"; + const std::string LOG_CONTINUE_REGEX = R"(\s+at\s.*)"; + + const std::string LOG_END_STRING = " ...23 more"; + const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more)"; + + const std::string LOG_UNMATCH = "unmatch log"; +}; + +// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginContinue); +// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginEnd); +// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); +// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithContinueEnd); +UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); + +void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { + Json::Value config; + config["EndPattern"] = LOG_END_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + // logFileReader.mDiscardUnmatch = true; + { // case: end with end + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + } + { // case: end with unmatch + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + } +} + } // namespace logtail UNIT_TEST_MAIN From b91c43f77774a2849dee32c0195d453ac0a26d41 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 02:36:45 +0000 Subject: [PATCH 02/12] fix --- core/reader/LogFileReader.cpp | 4 - .../RemoveLastIncompleteLogUnittest.cpp | 75 ++++++++++++------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index f777c0525f..b4b5a44c86 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2072,10 +2072,6 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll // Single line rollback or all unmatch rollback rollbackLineFeedCount = 0; if (buffer[size - 1] == '\n') { - if (mReaderConfig.first->mInputType == FileReaderOptions::InputType::InputFile) { - // 如果是文件采集，直接返回，节约性能 - return size; - } endPs = size - 1; } else { endPs = size; diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index f63183c515..92cca93dd6 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -437,24 +437,26 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { // case: all unmatch - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_UNMATCH; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); - } - { // case: all unmatch - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_UNMATCH + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + } } } @@ -575,17 +577,32 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { // case: all unmatch - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + } } { std::string expectMatch From 6fbfdc22204d31fabdba984a7f4953ec5f656a1d Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 02:37:46 +0000 Subject: [PATCH 03/12] fix --- core/reader/LogFileReader.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index b4b5a44c86..92e8361473 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2405,12 +2405,8 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - // containerd 不需要外层协议的 dataRaw - finalLine.data = line.data; - finalLine.fullLine = line.fullLine; - finalLine.lineBegin = line.lineBegin; - finalLine.rollbackLineFeedCount += line.rollbackLineFeedCount; - finalLine.lineEnd = line.lineEnd; + line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + finalLine = std::move(line); mergeLines(finalLine, finalLine, true); if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { From 1293212b264e67d91734c642e5ec454d870fed8b Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 05:11:53 +0000 Subject: [PATCH 04/12] fix --- core/reader/LogFileReader.cpp | 73 +++++++++++++++---- core/reader/LogFileReader.h | 8 +- .../RemoveLastIncompleteLogUnittest.cpp | 57 +++++++-------- 3 files changed, 93 insertions(+), 45 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index 92e8361473..0855316b6a 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2050,6 +2050,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll content.data.size(), *mMultilineConfig.first->GetEndPatternReg(), exception)) { + rollbackLineFeedCount += content.forceRollbackLineFeedCount; // Ensure the end line is complete if (buffer[content.lineEnd] == '\n') { return content.lineEnd + 1; @@ -2061,10 +2062,12 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll *mMultilineConfig.first->GetStartPatternReg(), exception)) { // start + continue, start + rollbackLineFeedCount += content.forceRollbackLineFeedCount; rollbackLineFeedCount += content.rollbackLineFeedCount; // Keep all the buffer if rollback all return content.lineBegin; } + rollbackLineFeedCount += content.forceRollbackLineFeedCount; rollbackLineFeedCount += content.rollbackLineFeedCount; endPs = content.lineBegin - 1; } @@ -2280,10 +2283,20 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex != 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } for (int32_t begin = end; begin > 0; --begin) { @@ -2292,14 +2305,16 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer, .lineBegin = begin, .lineEnd = end, .rollbackLineFeedCount = 1, - .fullLine = true}; + .fullLine = true, + .forceRollbackLineFeedCount = 0}; } } return {.data = StringView(buffer.data(), end), .lineBegin = 0, .lineEnd = end, .rollbackLineFeedCount = 1, - .fullLine = true}; + .fullLine = true, + .forceRollbackLineFeedCount = 0}; } LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, @@ -2308,11 +2323,21 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex == 0) { // 异常情况, DockerJsonFileParse不允许在最后一个解析器 - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } size_t nextProtocolFunctionIndex = protocolFunctionIndex - 1; @@ -2326,7 +2351,12 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + if (line.fullLine) { + line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + } else { + line.forceRollbackLineFeedCount += line.rollbackLineFeedCount; + } finalLine = std::move(line); if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { @@ -2334,7 +2364,8 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, - .fullLine = false}; + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } end = finalLine.lineBegin - 1; } @@ -2385,11 +2416,21 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex == 0) { - // 异常情况, DockerJsonFileParse不允许在最后一个解析器 - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + // 异常情况, ContainerdTextParser不允许在最后一个解析器 + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } LineInfo finalLine; finalLine.fullLine = false; @@ -2405,7 +2446,12 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + if (line.fullLine) { + line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + } else { + line.forceRollbackLineFeedCount += line.rollbackLineFeedCount; + } finalLine = std::move(line); mergeLines(finalLine, finalLine, true); if (!finalLine.fullLine) { @@ -2414,7 +2460,8 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, - .fullLine = false}; + .fullLine = false, + .forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount}; } end = finalLine.lineBegin - 1; } diff --git a/core/reader/LogFileReader.h b/core/reader/LogFileReader.h index 0a75f0856b..4eb6309ae3 100644 --- a/core/reader/LogFileReader.h +++ b/core/reader/LogFileReader.h @@ -34,7 +34,6 @@ #include "event/Event.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/FileServer.h" -#include "file_server/FileServer.h" #include "file_server/MultilineOptions.h" #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" @@ -57,16 +56,19 @@ struct LineInfo { int32_t lineEnd; int32_t rollbackLineFeedCount; bool fullLine; + int32_t forceRollbackLineFeedCount; LineInfo(StringView data = StringView(), int32_t lineBegin = 0, int32_t lineEnd = 0, int32_t rollbackLineFeedCount = 0, - bool fullLine = false) + bool fullLine = false, + int32_t forceRollbackLineFeedCount = 0) : data(data), lineBegin(lineBegin), lineEnd(lineEnd), rollbackLineFeedCount(rollbackLineFeedCount), - fullLine(fullLine) {} + fullLine(fullLine), + forceRollbackLineFeedCount(forceRollbackLineFeedCount) {} }; class BaseLineParse { diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 92cca93dd6..c52966c7c5 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -448,14 +448,14 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + std::string expectMatch = "\n\n" + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } } } @@ -487,7 +487,7 @@ void GetLastLineUnittest::TestGetLastLineEmpty() { LogFileReader logFileReader( "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(nullptr, &ctx)); auto lastLine = logFileReader.GetLastLine(const_cast(testLog.data()), testLog.size()); - APSARA_TEST_EQUAL_FATAL(0, lastLine.size()); + APSARA_TEST_EQUAL_FATAL(0, int(lastLine.size())); APSARA_TEST_EQUAL_FATAL("", std::string(lastLine.data(), lastLine.size())); APSARA_TEST_EQUAL_FATAL(testLog.data(), lastLine.data()); } @@ -547,7 +547,6 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - // logFileReader.mDiscardUnmatch = true; { // case: end with end std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; @@ -558,9 +557,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } { // case: end with unmatch std::string expectMatch @@ -572,9 +571,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); } { // case: all unmatch { @@ -586,38 +585,38 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); } { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } } { std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART - + LOG_UNMATCH + "\n"; + std::string testLog + = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } { std::string expectMatch @@ -630,9 +629,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog); - APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } } From b81b5c9301166793b9e73f6b907bc1211f469a9d Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 05:54:43 +0000 Subject: [PATCH 05/12] fix --- core/reader/LogFileReader.cpp | 13 +- .../RemoveLastIncompleteLogUnittest.cpp | 300 +++++++++++++++--- 2 files changed, 260 insertions(+), 53 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index 0855316b6a..263188c763 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2021,7 +2021,8 @@ LogFileReader::FileCompareResult LogFileReader::CompareToFile(const string& file 3. continue\nend\ncontinue\nend\n -> continue\nxxx\nend 5. mLogEndRegPtr != NULL 1. xxx\nend\n -> xxx\nend - 1. xxx\nend\nxxx\n -> xxx\nend + 2. xxx\nend\nxxx\n -> xxx\nend + 3. xxx\nend -> "" */ /* return: the number of bytes left, including \n @@ -2040,6 +2041,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll rollbackLineFeedCount = 0; if (mReaderConfig.first->mInputType == FileReaderOptions::InputType::InputContainerStdio) { // Multiline rollback + bool foundEnd = false; if (mMultilineConfig.first->IsMultiline()) { std::string exception; while (endPs >= 0) { @@ -2051,6 +2053,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll *mMultilineConfig.first->GetEndPatternReg(), exception)) { rollbackLineFeedCount += content.forceRollbackLineFeedCount; + foundEnd = true; // Ensure the end line is complete if (buffer[content.lineEnd] == '\n') { return content.lineEnd + 1; @@ -2072,6 +2075,9 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll endPs = content.lineBegin - 1; } } + if (mMultilineConfig.first->GetEndPatternReg() && foundEnd) { + return 0; + } // Single line rollback or all unmatch rollback rollbackLineFeedCount = 0; if (buffer[size - 1] == '\n') { @@ -2088,6 +2094,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll rollbackLineFeedCount = content.rollbackLineFeedCount; return content.lineBegin; } else { + bool foundEnd = false; // Multiline rollback if (mMultilineConfig.first->IsMultiline()) { std::string exception; @@ -2097,6 +2104,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll // start + end, continue + end, end if (BoostRegexSearch( content.data(), content.size(), *mMultilineConfig.first->GetEndPatternReg(), exception)) { + foundEnd = true; // Ensure the end line is complete if (buffer[endPs] == '\n') { return endPs + 1; @@ -2116,6 +2124,9 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll endPs = content.data() - buffer - 1; } } + if (mMultilineConfig.first->GetEndPatternReg() && foundEnd) { + return 0; + } // Single line rollback or all unmatch rollback rollbackLineFeedCount = 0; if (buffer[size - 1] == '\n') { diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index c52966c7c5..714248ad2f 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -417,14 +417,26 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with end - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + { + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string testLog = std::string(expectMatch.data()); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING; + std::string testLog = std::string(expectMatch.data()); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), ""); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } } { // case: end with unmatch std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; @@ -504,10 +516,8 @@ class ContainerdTextRemoveLastIncompleteLogMultilineUnittest : public ::testing: logFileReader->mLineParsers.emplace_back(baseLineParsePtr); return logFileReader; } - void TestRemoveLastIncompleteLogWithBeginContinue(); void TestRemoveLastIncompleteLogWithBeginEnd(); void TestRemoveLastIncompleteLogWithBegin(); - void TestRemoveLastIncompleteLogWithContinueEnd(); void TestRemoveLastIncompleteLogWithEnd(); void SetUp() override { readerOpts.mInputType = FileReaderOptions::InputType::InputContainerStdio; } @@ -522,21 +532,146 @@ class ContainerdTextRemoveLastIncompleteLogMultilineUnittest : public ::testing: const std::string LOG_BEGIN_STRING = "Exception in thread \"main\" java.lang.NullPointerException"; const std::string LOG_BEGIN_REGEX = R"(Exception.*)"; - const std::string LOG_CONTINUE_STRING = " at com.example.myproject.Book.getTitle(Book.java:16)"; - const std::string LOG_CONTINUE_REGEX = R"(\s+at\s.*)"; - const std::string LOG_END_STRING = " ...23 more"; - const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more)"; + const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more.*)"; const std::string LOG_UNMATCH = "unmatch log"; }; -// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginContinue); // UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginEnd); -// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); -// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithContinueEnd); +UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); +void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBegin() { + Json::Value config; + config["StartPattern"] = LOG_BEGIN_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with begin + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with unmatch + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + } + { // case: all unmatch + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with part log + } +} + void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { Json::Value config; config["EndPattern"] = LOG_END_REGEX; @@ -548,18 +683,62 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); { // case: end with end - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch; + { + std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART + + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH; + std::string testLog = expectMatch; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(4, rollbackLineFeedCount); + } + { + std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART + + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } } { // case: end with unmatch std::string expectMatch @@ -603,35 +782,52 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } } - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog - = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; + { // case: end with part log + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog + = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART - + LOG_UNMATCH + "\n"; + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + + LOG_UNMATCH + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + + "2021-08-25T07:00:00.000000000Z"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } } } From 60954b4ef4cb6c68f17fab749cbe5b687cba6de6 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 06:28:38 +0000 Subject: [PATCH 06/12] fix --- .../RemoveLastIncompleteLogUnittest.cpp | 70 ++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 714248ad2f..3a640952bc 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -538,10 +538,78 @@ class ContainerdTextRemoveLastIncompleteLogMultilineUnittest : public ::testing: const std::string LOG_UNMATCH = "unmatch log"; }; -// UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginEnd); +UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginEnd); UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); +void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBeginEnd() { + Json::Value config; + config["StartPattern"] = LOG_BEGIN_REGEX; + config["EndPattern"] = LOG_END_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with begin end + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = std::string(expectMatch.data()); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { // case: end with begin + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: end with unmatch + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } +} + void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBegin() { Json::Value config; config["StartPattern"] = LOG_BEGIN_REGEX; From e12f34e3d595979068415db3d931a68b7d914d09 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 07:28:52 +0000 Subject: [PATCH 07/12] fix --- core/reader/LogFileReader.cpp | 14 +++++---- .../RemoveLastIncompleteLogUnittest.cpp | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index 263188c763..b0bc3a9f2a 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2362,11 +2362,12 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; if (line.fullLine) { - line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount + line.rollbackLineFeedCount; } else { - line.forceRollbackLineFeedCount += line.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount + line.rollbackLineFeedCount; + line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount; } finalLine = std::move(line); if (!finalLine.fullLine) { @@ -2457,11 +2458,12 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); - line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; if (line.fullLine) { - line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount + line.rollbackLineFeedCount; } else { - line.forceRollbackLineFeedCount += line.rollbackLineFeedCount; + line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount + line.rollbackLineFeedCount; + line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount; } finalLine = std::move(line); mergeLines(finalLine, finalLine, true); diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 3a640952bc..94fddcbec4 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -737,6 +737,36 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom } } { // case: end with part log + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" + + LOG_PART + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" + + LOG_PART + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } } } From d28c242cde2552ff86f793341c4f1888efd7f5d1 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 07:39:44 +0000 Subject: [PATCH 08/12] fix --- core/reader/LogFileReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index b0bc3a9f2a..d2c488f674 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2377,7 +2377,7 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, .lineEnd = 0, .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, .fullLine = false, - .forceRollbackLineFeedCount = 0}; + .forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount}; } end = finalLine.lineBegin - 1; } From 6566ced41f0332b3b37530113bf3dd16ca5edcef Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 07:46:04 +0000 Subject: [PATCH 09/12] fix test --- .../reader/GetLastLineDataUnittest.cpp | 81 ++++++++++++------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/core/unittest/reader/GetLastLineDataUnittest.cpp b/core/unittest/reader/GetLastLineDataUnittest.cpp index a90ab57a8c..147340b549 100644 --- a/core/unittest/reader/GetLastLineDataUnittest.cpp +++ b/core/unittest/reader/GetLastLineDataUnittest.cpp @@ -104,7 +104,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -122,7 +123,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -197,7 +199,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -215,7 +218,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -289,7 +293,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } // case: F + P + P + '\n' @@ -308,7 +313,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } @@ -483,7 +489,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: P + P + '\n' @@ -502,7 +509,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } } @@ -532,7 +540,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(0, line.lineEnd); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } { @@ -571,7 +580,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -589,7 +599,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -664,7 +675,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -682,7 +694,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -756,7 +769,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } // case: F + P + P + '\n' @@ -775,7 +789,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } @@ -950,7 +965,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: P + P + '\n' @@ -969,7 +985,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } } @@ -1069,7 +1086,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1086,7 +1104,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1103,7 +1122,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1120,7 +1140,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1162,7 +1183,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1181,7 +1203,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1200,7 +1223,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1219,7 +1243,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1332,7 +1357,8 @@ void LastMatchedContainerdTextWithDockerJsonUnittest::TestContainerdTextWithDock endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(4, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(true, line.fullLine); @@ -1382,7 +1408,8 @@ void LastMatchedContainerdTextWithDockerJsonUnittest::TestDockerJsonWithContaine endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(true, line.fullLine); From 16dc3859123a61e08cc9862abe70370d70101e17 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 27 Dec 2024 08:55:18 +0000 Subject: [PATCH 10/12] add test --- .../RemoveLastIncompleteLogUnittest.cpp | 343 +++++++++++++++++- 1 file changed, 333 insertions(+), 10 deletions(-) diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 94fddcbec4..ca0077d359 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -14,6 +14,8 @@ #include "common/FileSystemUtil.h" #include "common/memory/SourceBuffer.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" #include "reader/LogFileReader.h" #include "unittest/Unittest.h" @@ -506,16 +508,6 @@ void GetLastLineUnittest::TestGetLastLineEmpty() { class ContainerdTextRemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { public: - std::shared_ptr GetLogFileReader(const Json::Value& config) { - MultilineOptions multilineOpts; - multilineOpts.Init(config, ctx, ""); - auto logFileReader = std::make_shared( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - BaseLineParse* baseLineParsePtr = nullptr; - baseLineParsePtr = logFileReader->GetParser(LogFileReader::BUFFER_SIZE); - logFileReader->mLineParsers.emplace_back(baseLineParsePtr); - return logFileReader; - } void TestRemoveLastIncompleteLogWithBeginEnd(); void TestRemoveLastIncompleteLogWithBegin(); void TestRemoveLastIncompleteLogWithEnd(); @@ -929,6 +921,337 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom } } +class DockerJsonRemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { +public: + void TestRemoveLastIncompleteLogWithBeginEnd(); + void TestRemoveLastIncompleteLogWithBegin(); + void TestRemoveLastIncompleteLogWithEnd(); + void SetUp() override { readerOpts.mInputType = FileReaderOptions::InputType::InputContainerStdio; } + +private: + FileReaderOptions readerOpts; + PipelineContext ctx; + + const std::string LOG_BEGIN_STRING = "Exception in thread \"main\" java.lang.NullPointerException"; + const std::string LOG_BEGIN_REGEX = R"(Exception.*)"; + + const std::string LOG_END_STRING = " ...23 more"; + const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more.*)"; + + const std::string LOG_UNMATCH = "unmatch log"; + + std::string BuildLog(const std::string& log, bool isNormalLog = true) { + if (isNormalLog) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.Key("log"); + writer.String((log + "\\n").c_str()); + writer.Key("stream"); + writer.String("stdout"); + writer.Key("time"); + writer.String("2024-02-19T03:49:37.793533014Z"); + writer.EndObject(); + return buffer.GetString(); + } else { + return R"({"log":")" + log + R"(\n","stream":"stdout","time":"2024-02-19T03:49:37.79)"; + } + } +}; + +UNIT_TEST_CASE(DockerJsonRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); +UNIT_TEST_CASE(DockerJsonRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); + +void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBegin() { + Json::Value config; + config["StartPattern"] = LOG_BEGIN_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(0); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with begin + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with unmatch + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + } + { // case: all unmatch + { + std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with part log + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) + + "\n" + BuildLog(LOG_BEGIN_STRING, false) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) + + "\n" + BuildLog(LOG_BEGIN_STRING, false); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } +} + +void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { + Json::Value config; + config["EndPattern"] = LOG_END_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(0); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with end + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING); + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + } + { // case: end with unmatch + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + } + { // case: end with part log + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + BuildLog(LOG_UNMATCH, false); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + BuildLog(LOG_UNMATCH, false) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + "2021-08-25T07:00:00.000000000Z"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } +} } // namespace logtail UNIT_TEST_MAIN From fa1f7d10dcf5dd16bbaf42df6d0de9250a5a0500 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 30 Dec 2024 06:11:27 +0000 Subject: [PATCH 11/12] add test --- core/reader/LogFileReader.cpp | 8 +++--- .../RemoveLastIncompleteLogUnittest.cpp | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index d2c488f674..a169107563 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2087,11 +2087,13 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll } LineInfo content = NewGetLastLine(StringView(buffer, size), endPs, true); // 最后一行是完整行,且以 \n 结尾 - if (content.fullLine && buffer[size - 1] == '\n' && content.lineEnd == endPs) { - return size; + if (content.fullLine && buffer[content.lineEnd] == '\n') { + rollbackLineFeedCount += content.forceRollbackLineFeedCount; + return content.lineEnd + 1; } content = NewGetLastLine(StringView(buffer, size), endPs, false); - rollbackLineFeedCount = content.rollbackLineFeedCount; + rollbackLineFeedCount += content.forceRollbackLineFeedCount; + rollbackLineFeedCount += content.rollbackLineFeedCount; return content.lineBegin; } else { bool foundEnd = false; diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index ca0077d359..f0848940bd 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -727,6 +727,32 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom APSARA_TEST_EQUAL(expectMatch, matchLog); APSARA_TEST_EQUAL(1, rollbackLineFeedCount); } + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } } { // case: end with part log { From 59969db67a6c49a594b935b68ef679dfee976baa Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 30 Dec 2024 12:51:09 +0000 Subject: [PATCH 12/12] fix --- core/reader/LogFileReader.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index a169107563..4d5ad09188 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -2364,14 +2364,19 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); + int32_t rollbackLineFeedCount = 0; + int32_t forceRollbackLineFeedCount = 0; if (line.fullLine) { - line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; - line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount + line.rollbackLineFeedCount; + rollbackLineFeedCount = line.rollbackLineFeedCount; + forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; } else { - line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount + line.rollbackLineFeedCount; - line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount; + forceRollbackLineFeedCount + = finalLine.forceRollbackLineFeedCount + line.forceRollbackLineFeedCount + line.rollbackLineFeedCount; + rollbackLineFeedCount = 0; } finalLine = std::move(line); + finalLine.rollbackLineFeedCount = rollbackLineFeedCount; + finalLine.forceRollbackLineFeedCount = forceRollbackLineFeedCount; if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { return {.data = StringView(), @@ -2460,14 +2465,19 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo line; parseLine(rawLine, line); + int32_t rollbackLineFeedCount = 0; + int32_t forceRollbackLineFeedCount = 0; if (line.fullLine) { - line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; - line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount + line.rollbackLineFeedCount; + rollbackLineFeedCount = line.rollbackLineFeedCount; + forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; } else { - line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount + line.rollbackLineFeedCount; - line.rollbackLineFeedCount = finalLine.rollbackLineFeedCount; + forceRollbackLineFeedCount + = finalLine.forceRollbackLineFeedCount + line.forceRollbackLineFeedCount + line.rollbackLineFeedCount; + rollbackLineFeedCount = 0; } finalLine = std::move(line); + finalLine.rollbackLineFeedCount = rollbackLineFeedCount; + finalLine.forceRollbackLineFeedCount = forceRollbackLineFeedCount; mergeLines(finalLine, finalLine, true); if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) {