diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index b20bbf71fb..4d5ad09188 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -50,7 +50,6 @@ #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" #include "monitor/MetricConstants.h" -#include "monitor/MetricConstants.h" #include "processor/inner/ProcessorParseContainerLogNative.h" #include "rapidjson/document.h" #include "reader/JsonLogFileReader.h" @@ -2022,7 +2021,8 @@ LogFileReader::FileCompareResult LogFileReader::CompareToFile(const string& file 3. continue\nend\ncontinue\nend\n -> continue\nxxx\nend 5. mLogEndRegPtr != NULL 1. xxx\nend\n -> xxx\nend - 1. xxx\nend\nxxx\n -> xxx\nend + 2. xxx\nend\nxxx\n -> xxx\nend + 3. xxx\nend -> "" */ /* return: the number of bytes left, including \n @@ -2041,6 +2041,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll rollbackLineFeedCount = 0; if (mReaderConfig.first->mInputType == FileReaderOptions::InputType::InputContainerStdio) { // Multiline rollback + bool foundEnd = false; if (mMultilineConfig.first->IsMultiline()) { std::string exception; while (endPs >= 0) { @@ -2051,6 +2052,8 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll content.data.size(), *mMultilineConfig.first->GetEndPatternReg(), exception)) { + rollbackLineFeedCount += content.forceRollbackLineFeedCount; + foundEnd = true; // Ensure the end line is complete if (buffer[content.lineEnd] == '\n') { return content.lineEnd + 1; @@ -2062,14 +2065,19 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll *mMultilineConfig.first->GetStartPatternReg(), exception)) { // start + continue, start + rollbackLineFeedCount += content.forceRollbackLineFeedCount; rollbackLineFeedCount += content.rollbackLineFeedCount; // Keep all the buffer if rollback all return content.lineBegin; } + rollbackLineFeedCount += content.forceRollbackLineFeedCount; rollbackLineFeedCount += content.rollbackLineFeedCount; endPs = content.lineBegin - 1; } } + if (mMultilineConfig.first->GetEndPatternReg() && foundEnd) { + return 0; + } // Single line rollback or all unmatch rollback rollbackLineFeedCount = 0; if (buffer[size - 1] == '\n') { @@ -2079,13 +2087,16 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll } LineInfo content = NewGetLastLine(StringView(buffer, size), endPs, true); // 最后一行是完整行,且以 \n 结尾 - if (content.fullLine && buffer[endPs] == '\n') { - return size; + if (content.fullLine && buffer[content.lineEnd] == '\n') { + rollbackLineFeedCount += content.forceRollbackLineFeedCount; + return content.lineEnd + 1; } content = NewGetLastLine(StringView(buffer, size), endPs, false); - rollbackLineFeedCount = content.rollbackLineFeedCount; + rollbackLineFeedCount += content.forceRollbackLineFeedCount; + rollbackLineFeedCount += content.rollbackLineFeedCount; return content.lineBegin; } else { + bool foundEnd = false; // Multiline rollback if (mMultilineConfig.first->IsMultiline()) { std::string exception; @@ -2095,6 +2106,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll // start + end, continue + end, end if (BoostRegexSearch( content.data(), content.size(), *mMultilineConfig.first->GetEndPatternReg(), exception)) { + foundEnd = true; // Ensure the end line is complete if (buffer[endPs] == '\n') { return endPs + 1; @@ -2114,6 +2126,9 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll endPs = content.data() - buffer - 1; } } + if (mMultilineConfig.first->GetEndPatternReg() && foundEnd) { + return 0; + } // Single line rollback or all unmatch rollback rollbackLineFeedCount = 0; if (buffer[size - 1] == '\n') { @@ -2281,26 +2296,38 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex != 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } for (int32_t begin = end; begin > 0; --begin) { - if (begin == 0 || buffer[begin - 1] == '\n') { + if (buffer[begin - 1] == '\n') { return {.data = StringView(buffer.data() + begin, end - begin), .lineBegin = begin, .lineEnd = end, .rollbackLineFeedCount = 1, - .fullLine = true}; + .fullLine = true, + .forceRollbackLineFeedCount = 0}; } } return {.data = StringView(buffer.data(), end), .lineBegin = 0, .lineEnd = end, .rollbackLineFeedCount = 1, - .fullLine = true}; + .fullLine = true, + .forceRollbackLineFeedCount = 0}; } LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, @@ -2309,11 +2336,21 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex == 0) { // 异常情况, DockerJsonFileParse不允许在最后一个解析器 - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } size_t nextProtocolFunctionIndex = protocolFunctionIndex - 1; @@ -2321,24 +2358,33 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, while (!finalLine.fullLine) { LineInfo rawLine = (*lineParsers)[nextProtocolFunctionIndex]->NewGetLastLine( buffer, end, nextProtocolFunctionIndex, needSingleLine, lineParsers); - if (rawLine.data.back() == '\n') { + if (rawLine.data.size() > 0 && rawLine.data.back() == '\n') { rawLine.data = StringView(rawLine.data.data(), rawLine.data.size() - 1); } LineInfo line; parseLine(rawLine, line); - finalLine.data = line.data; - finalLine.fullLine = line.fullLine; - finalLine.lineBegin = line.lineBegin; - finalLine.rollbackLineFeedCount += line.rollbackLineFeedCount; - finalLine.dataRaw = line.dataRaw; - if (finalLine.lineEnd == 0) { - finalLine.lineEnd = line.lineEnd; + int32_t rollbackLineFeedCount = 0; + int32_t forceRollbackLineFeedCount = 0; + if (line.fullLine) { + rollbackLineFeedCount = line.rollbackLineFeedCount; + forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + } else { + forceRollbackLineFeedCount + = finalLine.forceRollbackLineFeedCount + line.forceRollbackLineFeedCount + line.rollbackLineFeedCount; + rollbackLineFeedCount = 0; } + finalLine = std::move(line); + finalLine.rollbackLineFeedCount = rollbackLineFeedCount; + finalLine.forceRollbackLineFeedCount = forceRollbackLineFeedCount; if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { - finalLine.data = StringView(); - return finalLine; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, + .fullLine = false, + .forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount}; } end = finalLine.lineBegin - 1; } @@ -2349,7 +2395,9 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer, bool DockerJsonFileParser::parseLine(LineInfo rawLine, LineInfo& paseLine) { paseLine = rawLine; paseLine.fullLine = false; - + if (rawLine.data.size() == 0) { + return false; + } rapidjson::Document doc; doc.Parse(rawLine.data.data(), rawLine.data.size()); @@ -2387,39 +2435,58 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, bool needSingleLine, std::vector* lineParsers) { if (end == 0) { - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } if (protocolFunctionIndex == 0) { - // 异常情况, DockerJsonFileParse不允许在最后一个解析器 - return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false}; + // 异常情况, ContainerdTextParser不允许在最后一个解析器 + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = 0, + .fullLine = false, + .forceRollbackLineFeedCount = 0}; } LineInfo finalLine; finalLine.fullLine = false; - // 跳过最后的连续P size_t nextProtocolFunctionIndex = protocolFunctionIndex - 1; + // 跳过最后的连续P while (!finalLine.fullLine) { LineInfo rawLine = (*lineParsers)[nextProtocolFunctionIndex]->NewGetLastLine( buffer, end, nextProtocolFunctionIndex, needSingleLine, lineParsers); - if (rawLine.data.back() == '\n') { + if (rawLine.data.size() > 0 && rawLine.data.back() == '\n') { rawLine.data = StringView(rawLine.data.data(), rawLine.data.size() - 1); } LineInfo line; parseLine(rawLine, line); - // containerd 不需要外层协议的 dataRaw - finalLine.data = line.data; - finalLine.fullLine = line.fullLine; - finalLine.lineBegin = line.lineBegin; - finalLine.rollbackLineFeedCount += line.rollbackLineFeedCount; - mergeLines(finalLine, finalLine, true); - if (finalLine.lineEnd == 0) { - finalLine.lineEnd = line.lineEnd; + int32_t rollbackLineFeedCount = 0; + int32_t forceRollbackLineFeedCount = 0; + if (line.fullLine) { + rollbackLineFeedCount = line.rollbackLineFeedCount; + forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount; + } else { + forceRollbackLineFeedCount + = finalLine.forceRollbackLineFeedCount + line.forceRollbackLineFeedCount + line.rollbackLineFeedCount; + rollbackLineFeedCount = 0; } + finalLine = std::move(line); + finalLine.rollbackLineFeedCount = rollbackLineFeedCount; + finalLine.forceRollbackLineFeedCount = forceRollbackLineFeedCount; + mergeLines(finalLine, finalLine, true); if (!finalLine.fullLine) { if (finalLine.lineBegin == 0) { - finalLine.data = StringView(); - return finalLine; + return {.data = StringView(), + .lineBegin = 0, + .lineEnd = 0, + .rollbackLineFeedCount = finalLine.rollbackLineFeedCount, + .fullLine = false, + .forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount}; } end = finalLine.lineBegin - 1; } @@ -2442,7 +2509,7 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer, LineInfo previousLine; LineInfo rawLine = (*lineParsers)[nextProtocolFunctionIndex]->NewGetLastLine( buffer, finalLine.lineBegin - 1, nextProtocolFunctionIndex, needSingleLine, lineParsers); - if (rawLine.data.back() == '\n') { + if (rawLine.data.size() > 0 && rawLine.data.back() == '\n') { rawLine.data = StringView(rawLine.data.data(), rawLine.data.size() - 1); } @@ -2479,9 +2546,10 @@ void ContainerdTextParser::parseLine(LineInfo rawLine, LineInfo& paseLine) { const char* lineEnd = rawLine.data.data() + rawLine.data.size(); paseLine = rawLine; paseLine.fullLine = true; - + if (rawLine.data.size() == 0) { + return; + } // 寻找第一个分隔符位置 time - StringView timeValue; const char* pch1 = std::find(rawLine.data.data(), lineEnd, ProcessorParseContainerLogNative::CONTAINERD_DELIMITER); if (pch1 == lineEnd) { return; @@ -2497,6 +2565,10 @@ void ContainerdTextParser::parseLine(LineInfo rawLine, LineInfo& paseLine) { return; } // 如果既不以 P 开头,也不以 F 开头 + if (pch2 + 1 >= lineEnd) { + paseLine.data = StringView(pch2 + 1, lineEnd - pch2 - 1); + return; + } if (*(pch2 + 1) != ProcessorParseContainerLogNative::CONTAINERD_PART_TAG && *(pch2 + 1) != ProcessorParseContainerLogNative::CONTAINERD_FULL_TAG) { paseLine.data = StringView(pch2 + 1, lineEnd - pch2 - 1); diff --git a/core/reader/LogFileReader.h b/core/reader/LogFileReader.h index 0a75f0856b..4eb6309ae3 100644 --- a/core/reader/LogFileReader.h +++ b/core/reader/LogFileReader.h @@ -34,7 +34,6 @@ #include "event/Event.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/FileServer.h" -#include "file_server/FileServer.h" #include "file_server/MultilineOptions.h" #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" @@ -57,16 +56,19 @@ struct LineInfo { int32_t lineEnd; int32_t rollbackLineFeedCount; bool fullLine; + int32_t forceRollbackLineFeedCount; LineInfo(StringView data = StringView(), int32_t lineBegin = 0, int32_t lineEnd = 0, int32_t rollbackLineFeedCount = 0, - bool fullLine = false) + bool fullLine = false, + int32_t forceRollbackLineFeedCount = 0) : data(data), lineBegin(lineBegin), lineEnd(lineEnd), rollbackLineFeedCount(rollbackLineFeedCount), - fullLine(fullLine) {} + fullLine(fullLine), + forceRollbackLineFeedCount(forceRollbackLineFeedCount) {} }; class BaseLineParse { diff --git a/core/unittest/reader/GetLastLineDataUnittest.cpp b/core/unittest/reader/GetLastLineDataUnittest.cpp index 8ed6cd2f33..147340b549 100644 --- a/core/unittest/reader/GetLastLineDataUnittest.cpp +++ b/core/unittest/reader/GetLastLineDataUnittest.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include "common/FileSystemUtil.h" -#include "reader/LogFileReader.h" #include "common/memory/SourceBuffer.h" +#include "reader/LogFileReader.h" #include "unittest/Unittest.h" namespace logtail { @@ -104,7 +104,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -122,7 +123,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -197,7 +199,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -215,7 +218,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -289,7 +293,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } // case: F + P + P + '\n' @@ -308,7 +313,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } @@ -483,7 +489,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: P + P + '\n' @@ -502,7 +509,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } } @@ -516,6 +524,45 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { + { + std::string testLog = "\n2024-01-05T23:28:06.818486411+08:00 stdout P 123123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.lineEnd); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + { + std::string testLog = "\n2024-01-05T23:28:06.818486411+08:00 stdout F 123123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123123", line.data.to_string()); + APSARA_TEST_EQUAL(1, line.lineBegin); + APSARA_TEST_EQUAL(endPs, line.lineEnd); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + } // 异常情况+有回车 { // case: PartLogFlag存在，第三个空格存在但空格后无内容 @@ -533,7 +580,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -551,7 +599,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -626,7 +675,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag存在，第三个空格不存在 @@ -644,7 +694,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: PartLogFlag不存在，第二个空格存在 @@ -718,7 +769,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } // case: F + P + P + '\n' @@ -737,7 +789,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("789", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } @@ -912,7 +965,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } // case: P + P + '\n' @@ -931,7 +985,8 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(false, line.fullLine); } } @@ -1031,7 +1086,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1048,7 +1104,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1065,7 +1122,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1082,7 +1140,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1124,7 +1183,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1143,7 +1203,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1162,7 +1223,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1181,7 +1243,8 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(false, line.fullLine); @@ -1294,7 +1357,8 @@ void LastMatchedContainerdTextWithDockerJsonUnittest::TestContainerdTextWithDock endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(4, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(true, line.fullLine); @@ -1344,7 +1408,8 @@ void LastMatchedContainerdTextWithDockerJsonUnittest::TestDockerJsonWithContaine endPs = size; } LineInfo line = logFileReader.NewGetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); APSARA_TEST_EQUAL(true, line.fullLine); diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 19e20463e2..f0848940bd 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -14,6 +14,8 @@ #include "common/FileSystemUtil.h" #include "common/memory/SourceBuffer.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" #include "reader/LogFileReader.h" #include "unittest/Unittest.h" @@ -417,14 +419,26 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with end - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + { + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string testLog = std::string(expectMatch.data()); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING; + std::string testLog = std::string(expectMatch.data()); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), ""); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } } { // case: end with unmatch std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; @@ -437,14 +451,26 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { // case: all unmatch - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_UNMATCH; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } } } @@ -475,11 +501,783 @@ void GetLastLineUnittest::TestGetLastLineEmpty() { LogFileReader logFileReader( "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(nullptr, &ctx)); auto lastLine = logFileReader.GetLastLine(const_cast(testLog.data()), testLog.size()); - APSARA_TEST_EQUAL_FATAL(0, lastLine.size()); + APSARA_TEST_EQUAL_FATAL(0, int(lastLine.size())); APSARA_TEST_EQUAL_FATAL("", std::string(lastLine.data(), lastLine.size())); APSARA_TEST_EQUAL_FATAL(testLog.data(), lastLine.data()); } +class ContainerdTextRemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { +public: + void TestRemoveLastIncompleteLogWithBeginEnd(); + void TestRemoveLastIncompleteLogWithBegin(); + void TestRemoveLastIncompleteLogWithEnd(); + void SetUp() override { readerOpts.mInputType = FileReaderOptions::InputType::InputContainerStdio; } + +private: + FileReaderOptions readerOpts; + PipelineContext ctx; + const std::string LOG_PART = "2021-08-25T07:00:00.000000000Z stdout P "; + const std::string LOG_FULL = "2021-08-25T07:00:00.000000000Z stdout F "; + const std::string LOG_FULL_NOT_FOUND = "2021-08-25T07:00:00.000000000Z stdout "; + const std::string LOG_ERROR = "2021-08-25T07:00:00.000000000Z stdout"; + + const std::string LOG_BEGIN_STRING = "Exception in thread \"main\" java.lang.NullPointerException"; + const std::string LOG_BEGIN_REGEX = R"(Exception.*)"; + + const std::string LOG_END_STRING = " ...23 more"; + const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more.*)"; + + const std::string LOG_UNMATCH = "unmatch log"; +}; + +UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginEnd); +UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); +UNIT_TEST_CASE(ContainerdTextRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); + +void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBeginEnd() { + Json::Value config; + config["StartPattern"] = LOG_BEGIN_REGEX; + config["EndPattern"] = LOG_END_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with begin end + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = std::string(expectMatch.data()); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { // case: end with begin + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: end with unmatch + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } +} + +void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBegin() { + Json::Value config; + config["StartPattern"] = LOG_BEGIN_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with begin + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with unmatch + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + } + { // case: all unmatch + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + } + { // case: end with part log + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" + + LOG_PART + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" + + LOG_PART + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } +} + +void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { + Json::Value config; + config["EndPattern"] = LOG_END_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with end + { + std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART + + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(4, rollbackLineFeedCount); + } + { + std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART + + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } + { // case: end with unmatch + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + } + { // case: end with part log + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog + = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + + "2021-08-25T07:00:00.000000000Z"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } +} + +class DockerJsonRemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { +public: + void TestRemoveLastIncompleteLogWithBeginEnd(); + void TestRemoveLastIncompleteLogWithBegin(); + void TestRemoveLastIncompleteLogWithEnd(); + void SetUp() override { readerOpts.mInputType = FileReaderOptions::InputType::InputContainerStdio; } + +private: + FileReaderOptions readerOpts; + PipelineContext ctx; + + const std::string LOG_BEGIN_STRING = "Exception in thread \"main\" java.lang.NullPointerException"; + const std::string LOG_BEGIN_REGEX = R"(Exception.*)"; + + const std::string LOG_END_STRING = " ...23 more"; + const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more.*)"; + + const std::string LOG_UNMATCH = "unmatch log"; + + std::string BuildLog(const std::string& log, bool isNormalLog = true) { + if (isNormalLog) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.Key("log"); + writer.String((log + "\\n").c_str()); + writer.Key("stream"); + writer.String("stdout"); + writer.Key("time"); + writer.String("2024-02-19T03:49:37.793533014Z"); + writer.EndObject(); + return buffer.GetString(); + } else { + return R"({"log":")" + log + R"(\n","stream":"stdout","time":"2024-02-19T03:49:37.79)"; + } + } +}; + +UNIT_TEST_CASE(DockerJsonRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); +UNIT_TEST_CASE(DockerJsonRemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); + +void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBegin() { + Json::Value config; + config["StartPattern"] = LOG_BEGIN_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(0); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with begin + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with unmatch + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); + } + } + { // case: all unmatch + { + std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + } + { // case: end with part log + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) + + "\n" + BuildLog(LOG_BEGIN_STRING, false) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) + + "\n" + BuildLog(LOG_BEGIN_STRING, false); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } +} + +void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { + Json::Value config; + config["EndPattern"] = LOG_END_REGEX; + MultilineOptions multilineOpts; + multilineOpts.Init(config, ctx, ""); + LogFileReader logFileReader( + "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + BaseLineParse* baseLineParsePtr = nullptr; + baseLineParsePtr = logFileReader.GetParser(0); + logFileReader.mLineParsers.emplace_back(baseLineParsePtr); + { // case: end with end + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING); + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + } + { // case: end with unmatch + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + } + { + std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); + } + } + { // case: end with part log + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + BuildLog(LOG_UNMATCH, false); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + BuildLog(LOG_UNMATCH, false) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + "2021-08-25T07:00:00.000000000Z"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); + } + } +} } // namespace logtail UNIT_TEST_MAIN