Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@quzard
quzard committed Dec 27, 2024
1 parent 6fbfdc2 commit 1293212
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 45 deletions.
73 changes: 60 additions & 13 deletions core/reader/LogFileReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2050,6 +2050,7 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll
content.data.size(),
*mMultilineConfig.first->GetEndPatternReg(),
exception)) {
rollbackLineFeedCount += content.forceRollbackLineFeedCount;
// Ensure the end line is complete
if (buffer[content.lineEnd] == '\n') {
return content.lineEnd + 1;
Expand All @@ -2061,10 +2062,12 @@ LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& roll
*mMultilineConfig.first->GetStartPatternReg(),
exception)) {
// start + continue, start
rollbackLineFeedCount += content.forceRollbackLineFeedCount;
rollbackLineFeedCount += content.rollbackLineFeedCount;
// Keep all the buffer if rollback all
return content.lineBegin;
}
rollbackLineFeedCount += content.forceRollbackLineFeedCount;
rollbackLineFeedCount += content.rollbackLineFeedCount;
endPs = content.lineBegin - 1;
}
Expand Down Expand Up @@ -2280,10 +2283,20 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer,
bool needSingleLine,
std::vector<BaseLineParse*>* lineParsers) {
if (end == 0) {
return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false};
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = 0,
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}
if (protocolFunctionIndex != 0) {
return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false};
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = 0,
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}

for (int32_t begin = end; begin > 0; --begin) {
Expand All @@ -2292,14 +2305,16 @@ LineInfo RawTextParser::NewGetLastLine(StringView buffer,
.lineBegin = begin,
.lineEnd = end,
.rollbackLineFeedCount = 1,
.fullLine = true};
.fullLine = true,
.forceRollbackLineFeedCount = 0};
}
}
return {.data = StringView(buffer.data(), end),
.lineBegin = 0,
.lineEnd = end,
.rollbackLineFeedCount = 1,
.fullLine = true};
.fullLine = true,
.forceRollbackLineFeedCount = 0};
}

LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer,
Expand All @@ -2308,11 +2323,21 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer,
bool needSingleLine,
std::vector<BaseLineParse*>* lineParsers) {
if (end == 0) {
return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false};
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = 0,
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}
if (protocolFunctionIndex == 0) {
// 异常情况, DockerJsonFileParse不允许在最后一个解析器
return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false};
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = 0,
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}

size_t nextProtocolFunctionIndex = protocolFunctionIndex - 1;
Expand All @@ -2326,15 +2351,21 @@ LineInfo DockerJsonFileParser::NewGetLastLine(StringView buffer,

LineInfo line;
parseLine(rawLine, line);
line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount;
line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount;
if (line.fullLine) {
line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount;
} else {
line.forceRollbackLineFeedCount += line.rollbackLineFeedCount;
}
finalLine = std::move(line);
if (!finalLine.fullLine) {
if (finalLine.lineBegin == 0) {
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = finalLine.rollbackLineFeedCount,
.fullLine = false};
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}
end = finalLine.lineBegin - 1;
}
Expand Down Expand Up @@ -2385,11 +2416,21 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer,
bool needSingleLine,
std::vector<BaseLineParse*>* lineParsers) {
if (end == 0) {
return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false};
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = 0,
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}
if (protocolFunctionIndex == 0) {
// 异常情况, DockerJsonFileParse不允许在最后一个解析器
return {.data = StringView(), .lineBegin = 0, .lineEnd = 0, .rollbackLineFeedCount = 0, .fullLine = false};
// 异常情况, ContainerdTextParser不允许在最后一个解析器
return {.data = StringView(),
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = 0,
.fullLine = false,
.forceRollbackLineFeedCount = 0};
}
LineInfo finalLine;
finalLine.fullLine = false;
Expand All @@ -2405,7 +2446,12 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer,

LineInfo line;
parseLine(rawLine, line);
line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount;
line.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount;
if (line.fullLine) {
line.rollbackLineFeedCount += finalLine.rollbackLineFeedCount;
} else {
line.forceRollbackLineFeedCount += line.rollbackLineFeedCount;
}
finalLine = std::move(line);
mergeLines(finalLine, finalLine, true);
if (!finalLine.fullLine) {
Expand All @@ -2414,7 +2460,8 @@ LineInfo ContainerdTextParser::NewGetLastLine(StringView buffer,
.lineBegin = 0,
.lineEnd = 0,
.rollbackLineFeedCount = finalLine.rollbackLineFeedCount,
.fullLine = false};
.fullLine = false,
.forceRollbackLineFeedCount = finalLine.forceRollbackLineFeedCount};
}
end = finalLine.lineBegin - 1;
}
Expand Down
8 changes: 5 additions & 3 deletions core/reader/LogFileReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
#include "event/Event.h"
#include "file_server/FileDiscoveryOptions.h"
#include "file_server/FileServer.h"
#include "file_server/FileServer.h"
#include "file_server/MultilineOptions.h"
#include "log_pb/sls_logs.pb.h"
#include "logger/Logger.h"
Expand All @@ -57,16 +56,19 @@ struct LineInfo {
int32_t lineEnd;
int32_t rollbackLineFeedCount;
bool fullLine;
int32_t forceRollbackLineFeedCount;
LineInfo(StringView data = StringView(),
int32_t lineBegin = 0,
int32_t lineEnd = 0,
int32_t rollbackLineFeedCount = 0,
bool fullLine = false)
bool fullLine = false,
int32_t forceRollbackLineFeedCount = 0)
: data(data),
lineBegin(lineBegin),
lineEnd(lineEnd),
rollbackLineFeedCount(rollbackLineFeedCount),
fullLine(fullLine) {}
fullLine(fullLine),
forceRollbackLineFeedCount(forceRollbackLineFeedCount) {}
};

class BaseLineParse {
Expand Down
57 changes: 28 additions & 29 deletions core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,14 +448,14 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn
APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount);
}
{
std::string expectMatch = "\n\n";
std::string testLog = expectMatch + LOG_UNMATCH + "\n";
std::string expectMatch = "\n\n" + LOG_UNMATCH + "\n";
std::string testLog = expectMatch;
int32_t rollbackLineFeedCount = 0;
int32_t matchSize = logFileReader.RemoveLastIncompleteLog(
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
APSARA_TEST_EQUAL_FATAL(static_cast<int32_t>(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch);
APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount);
APSARA_TEST_EQUAL(static_cast<int32_t>(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), expectMatch);
APSARA_TEST_EQUAL(0, rollbackLineFeedCount);
}
}
}
Expand Down Expand Up @@ -487,7 +487,7 @@ void GetLastLineUnittest::TestGetLastLineEmpty() {
LogFileReader logFileReader(
"dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(nullptr, &ctx));
auto lastLine = logFileReader.GetLastLine(const_cast<char*>(testLog.data()), testLog.size());
APSARA_TEST_EQUAL_FATAL(0, lastLine.size());
APSARA_TEST_EQUAL_FATAL(0, int(lastLine.size()));
APSARA_TEST_EQUAL_FATAL("", std::string(lastLine.data(), lastLine.size()));
APSARA_TEST_EQUAL_FATAL(testLog.data(), lastLine.data());
}
Expand Down Expand Up @@ -547,7 +547,6 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom
BaseLineParse* baseLineParsePtr = nullptr;
baseLineParsePtr = logFileReader.GetParser<ContainerdTextParser>(LogFileReader::BUFFER_SIZE);
logFileReader.mLineParsers.emplace_back(baseLineParsePtr);
// logFileReader.mDiscardUnmatch = true;
{ // case: end with end
std::string expectMatch
= LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n';
Expand All @@ -558,9 +557,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
const auto& matchLog = std::string(testLog.data(), matchSize);

APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog);
APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount);
APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(expectMatch, matchLog);
APSARA_TEST_EQUAL(0, rollbackLineFeedCount);
}
{ // case: end with unmatch
std::string expectMatch
Expand All @@ -572,9 +571,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
const auto& matchLog = std::string(testLog.data(), matchSize);

APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog);
APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount);
APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(expectMatch, matchLog);
APSARA_TEST_EQUAL(1, rollbackLineFeedCount);
}
{ // case: all unmatch
{
Expand All @@ -586,38 +585,38 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
const auto& matchLog = std::string(testLog.data(), matchSize);

APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog);
APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount);
APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(expectMatch, matchLog);
APSARA_TEST_EQUAL(1, rollbackLineFeedCount);
}
{
std::string expectMatch = "\n\n";
std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n";
std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n";
std::string testLog = expectMatch;

int32_t rollbackLineFeedCount = 0;
int32_t matchSize = logFileReader.RemoveLastIncompleteLog(
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
const auto& matchLog = std::string(testLog.data(), matchSize);

APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog);
APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount);
APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(expectMatch, matchLog);
APSARA_TEST_EQUAL(0, rollbackLineFeedCount);
}
}
{
std::string expectMatch
= LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n';
std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART
+ LOG_UNMATCH + "\n";
std::string testLog
= expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH;

int32_t rollbackLineFeedCount = 0;
int32_t matchSize = logFileReader.RemoveLastIncompleteLog(
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
const auto& matchLog = std::string(testLog.data(), matchSize);

APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog);
APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount);
APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(expectMatch, matchLog);
APSARA_TEST_EQUAL(3, rollbackLineFeedCount);
}
{
std::string expectMatch
Expand All @@ -630,9 +629,9 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom
const_cast<char*>(testLog.data()), testLog.size(), rollbackLineFeedCount);
const auto& matchLog = std::string(testLog.data(), matchSize);

APSARA_TEST_EQUAL_FATAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL_FATAL(expectMatch, matchLog);
APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount);
APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize);
APSARA_TEST_EQUAL(expectMatch, matchLog);
APSARA_TEST_EQUAL(3, rollbackLineFeedCount);
}
}

Expand Down

0 comments on commit 1293212

Please sign in to comment.