Skip to content

Commit

Permalink
polish code
Browse files Browse the repository at this point in the history
  • Loading branch information
henryzhx8 committed Dec 4, 2023
1 parent 3426d6d commit c3d0a9a
Show file tree
Hide file tree
Showing 34 changed files with 101 additions and 84 deletions.
2 changes: 1 addition & 1 deletion core/common/BoostRegexValidator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include <stdio.h>
#include <boost/regex.hpp>
#include <boost/algorithm/string.hpp>
#include "json/json.h"
#include <json/json.h>

using namespace std;

Expand Down
2 changes: 1 addition & 1 deletion core/common/JsonUtil.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#include <cstdint>
#include <string>

#include "json/json.h"
#include <json/json.h>

namespace logtail {

Expand Down
2 changes: 1 addition & 1 deletion core/common/YamlUtil.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

#include <string>

#include "json/json.h"
#include <json/json.h>
#include "yaml-cpp/yaml.h"

namespace logtail {
Expand Down
2 changes: 1 addition & 1 deletion core/config/provider/CommonConfigProvider.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#include <iostream>
#include <random>

#include "json/json.h"
#include <json/json.h>

#include "app_config/AppConfig.h"
#include "application/Application.h"
Expand Down
2 changes: 1 addition & 1 deletion core/container_manager/ContainerDiscoveryOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include <unordered_map>
#include <utility>

#include "json/json.h"
#include <json/json.h>

#include "pipeline/PipelineContext.h"

Expand Down
2 changes: 1 addition & 1 deletion core/file_server/DockerContainerPath.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
#include <unordered_map>
#include <vector>

#include "json/json.h"
#include <json/json.h>

#include "log_pb/sls_logs.pb.h"

Expand Down
2 changes: 1 addition & 1 deletion core/file_server/FileDiscoveryOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
#include <utility>
#include <vector>

#include "json/json.h"
#include <json/json.h>

#include "file_server/DockerContainerPath.h"
#include "pipeline/PipelineContext.h"
Expand Down
2 changes: 1 addition & 1 deletion core/file_server/MultilineOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#include <string>
#include <utility>

#include "json/json.h"
#include <json/json.h>
#include "boost/regex.hpp"

#include "pipeline/PipelineContext.h"
Expand Down
2 changes: 1 addition & 1 deletion core/flusher/FlusherSLS.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
#include <unordered_set>
#include <vector>

#include "json/json.h"
#include <json/json.h>

#include "common/LogstoreFeedbackKey.h"

Expand Down
2 changes: 1 addition & 1 deletion core/go_pipeline/LogtailPlugin.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
#include <unordered_map>
#include <utility>

#include "json/json.h"
#include <json/json.h>

#include "flusher/FlusherSLS.h"
#include "log_pb/sls_logs.pb.h"
Expand Down
2 changes: 1 addition & 1 deletion core/observer/network/NetworkConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include <sstream>
#include "LogtailAlarm.h"
#include "logger/Logger.h"
#include "json/json.h"
#include <json/json.h>
#include "common/JsonUtil.h"
#include "ExceptionBase.h"
#include "input/InputObserverNetwork.h"
Expand Down
2 changes: 1 addition & 1 deletion core/pipeline/GlobalConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include <string>
#include <unordered_set>

#include "json/json.h"
#include <json/json.h>

namespace logtail {

Expand Down
2 changes: 1 addition & 1 deletion core/pipeline/Pipeline.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
#include <memory>
#include <unordered_map>

#include "json/json.h"
#include <json/json.h>

#include "config/Config.h"
#include "models/PipelineEventGroup.h"
Expand Down
2 changes: 1 addition & 1 deletion core/plugin/instance/FlusherInstance.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

#include <memory>

#include "json/json.h"
#include <json/json.h>

#include "plugin/instance/PluginInstance.h"
#include "plugin/interface/Flusher.h"
Expand Down
2 changes: 1 addition & 1 deletion core/plugin/instance/InputInstance.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

#include <memory>

#include "json/json.h"
#include <json/json.h>

#include "plugin/instance/PluginInstance.h"
#include "plugin/interface/Input.h"
Expand Down
2 changes: 1 addition & 1 deletion core/plugin/instance/ProcessorInstance.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

#include <memory>

#include "json/json.h"
#include <json/json.h>

#include "models/PipelineEventGroup.h"
#include "monitor/LogtailMetric.h"
Expand Down
2 changes: 1 addition & 1 deletion core/plugin/interface/Flusher.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

#pragma once

#include "json/json.h"
#include <json/json.h>

#include "plugin/interface/Plugin.h"

Expand Down
2 changes: 1 addition & 1 deletion core/plugin/interface/Input.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

#pragma once

#include "json/json.h"
#include <json/json.h>

#include "plugin/interface/Plugin.h"

Expand Down
4 changes: 3 additions & 1 deletion core/plugin/interface/Processor.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@

#pragma once

#include "json/json.h"
#include <json/json.h>

#include "models/PipelineEventGroup.h"
#include "models/PipelineEventPtr.h"
#include "plugin/interface/Plugin.h"

namespace logtail {
Expand Down
27 changes: 14 additions & 13 deletions core/processor/ProcessorSplitLogStringNative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,37 +16,38 @@

#include "processor/ProcessorSplitLogStringNative.h"

#include <string>

#include "common/Constants.h"
#include "common/ParamExtractor.h"
#include "models/LogEvent.h"
#include "monitor/MetricConstants.h"
#include "plugin/instance/ProcessorInstance.h"

namespace logtail {

const std::string ProcessorSplitLogStringNative::sName = "processor_split_string_native";

bool ProcessorSplitLogStringNative::Init(const Json::Value& config) {
std::string errorMsg;
// SplitKey
if (!GetOptionalStringParam(config, "SplitKey", mSplitKey, errorMsg)) {
PARAM_WARNING_DEFAULT(mContext->GetLogger(), errorMsg, mSplitKey, sName, mContext->GetConfigName());

// SourceKey
if (!GetOptionalStringParam(config, "SourceKey", mSourceKey, errorMsg)) {
PARAM_WARNING_DEFAULT(mContext->GetLogger(), errorMsg, mSourceKey, sName, mContext->GetConfigName());
}
// Compatible with old logic.

// SplitChar
int32_t splitter = '\n';
if (!GetOptionalIntParam(config, "SplitChar", splitter, errorMsg)) {
PARAM_WARNING_DEFAULT(mContext->GetLogger(), errorMsg, "\\n", sName, mContext->GetConfigName());
} else {
mSplitChar = static_cast<char>(splitter);
}

// AppendingLogPositionMeta
if (!GetOptionalBoolParam(config, "AppendingLogPositionMeta", mAppendingLogPositionMeta, errorMsg)) {
PARAM_WARNING_DEFAULT(
mContext->GetLogger(), errorMsg, mAppendingLogPositionMeta, sName, mContext->GetConfigName());
}

mFeedLines = &(GetContext().GetProcessProfile().feedLines);
mSplitLines = &(GetContext().GetProcessProfile().splitLines);

return true;
}

Expand Down Expand Up @@ -76,11 +77,11 @@ void ProcessorSplitLogStringNative::ProcessEvent(PipelineEventGroup& logGroup,
return;
}
const LogEvent& sourceEvent = e.Cast<LogEvent>();
if (!sourceEvent.HasContent(mSplitKey)) {
if (!sourceEvent.HasContent(mSourceKey)) {
newEvents.emplace_back(e);
return;
}
StringView sourceVal = sourceEvent.GetContent(mSplitKey);
StringView sourceVal = sourceEvent.GetContent(mSourceKey);
std::vector<StringView> logIndex; // all splitted logs
int feedLines = 0;
LogSplit(sourceVal.data(), sourceVal.size(), feedLines, logIndex);
Expand All @@ -90,7 +91,7 @@ void ProcessorSplitLogStringNative::ProcessEvent(PipelineEventGroup& logGroup,
if (sourceEvent.HasContent(LOG_RESERVED_KEY_FILE_OFFSET)) {
sourceoffset = atol(sourceEvent.GetContent(LOG_RESERVED_KEY_FILE_OFFSET).data()); // use safer method
}
StringBuffer splitKey = logGroup.GetSourceBuffer()->CopyString(mSplitKey);
StringBuffer splitKey = logGroup.GetSourceBuffer()->CopyString(mSourceKey);
for (auto& content : logIndex) {
std::unique_ptr<LogEvent> targetEvent = LogEvent::CreateEvent(logGroup.GetSourceBuffer());
targetEvent->SetTimestamp(
Expand All @@ -104,7 +105,7 @@ void ProcessorSplitLogStringNative::ProcessEvent(PipelineEventGroup& logGroup,
}
if (sourceEvent.GetContents().size() > 1) { // copy other fields
for (auto& kv : sourceEvent.GetContents()) {
if (kv.first != mSplitKey && kv.first != LOG_RESERVED_KEY_FILE_OFFSET) {
if (kv.first != mSourceKey && kv.first != LOG_RESERVED_KEY_FILE_OFFSET) {
targetEvent->SetContentNoCopy(kv.first, kv.second);
}
}
Expand Down
62 changes: 34 additions & 28 deletions core/processor/ProcessorSplitLogStringNative.h
Original file line number Diff line number Diff line change
@@ -1,48 +1,54 @@
/*
* Copyright 2023 iLogtail Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Copyright 2023 iLogtail Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <cstdint>
#include <vector>

#include "common/Constants.h"
#include "plugin/interface/Processor.h"

namespace logtail {

class ProcessorSplitLogStringNative : public Processor {
public:
static const std::string sName;
static const std::string sName;

std::string mSplitKey = DEFAULT_CONTENT_KEY;
char mSplitChar = '\n';
bool mAppendingLogPositionMeta = false;
std::string mSourceKey = DEFAULT_CONTENT_KEY;
char mSplitChar = '\n';
bool mAppendingLogPositionMeta = false;

const std::string& Name() const override { return sName; }
bool Init(const Json::Value& config) override;
void Process(PipelineEventGroup& logGroup) override;
const std::string& Name() const override { return sName; }
bool Init(const Json::Value& config) override;
void Process(PipelineEventGroup& logGroup) override;

protected:
bool IsSupportedEvent(const PipelineEventPtr& e) const override;
bool IsSupportedEvent(const PipelineEventPtr& e) const override;

private:
void ProcessEvent(PipelineEventGroup& logGroup, const PipelineEventPtr& e, EventsContainer& newEvents);
void LogSplit(const char* buffer, int32_t size, int32_t& lineFeed, std::vector<StringView>& logIndex);
int* mFeedLines = nullptr;
int* mSplitLines = nullptr;
void ProcessEvent(PipelineEventGroup& logGroup, const PipelineEventPtr& e, EventsContainer& newEvents);
void LogSplit(const char* buffer, int32_t size, int32_t& lineFeed, std::vector<StringView>& logIndex);

int* mFeedLines = nullptr;
int* mSplitLines = nullptr;

#ifdef APSARA_UNIT_TEST_MAIN
friend class ProcessorRegexStringNativeUnittest;
friend class ProcessorParseDelimiterNativeUnittest;
friend class ProcessorRegexStringNativeUnittest;
friend class ProcessorParseDelimiterNativeUnittest;
#endif
};

Expand Down
Loading

0 comments on commit c3d0a9a

Please sign in to comment.