From dfed5d91781f9b1089b00d0ef7f679ec72e98195 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 10 Jan 2025 11:08:00 +0800 Subject: [PATCH] fix --- core/common/CircularBuffer.h | 1 - core/common/ParamExtractor.cpp | 55 +++ core/common/ParamExtractor.h | 14 +- core/common/TagConstants.h | 83 ---- core/constants/TagConstants.cpp | 36 ++ core/constants/TagConstants.h | 34 ++ core/file_server/ContainerInfo.cpp | 63 ++- core/file_server/ContainerInfo.h | 38 +- core/file_server/FileTagOptions.cpp | 132 +++--- core/file_server/FileTagOptions.h | 22 +- core/file_server/reader/LogFileReader.cpp | 98 ++--- core/file_server/reader/LogFileReader.h | 21 +- core/models/PipelineEventGroup.cpp | 7 - core/models/PipelineEventGroup.h | 7 +- core/pipeline/GlobalConfig.cpp | 45 +- core/pipeline/GlobalConfig.h | 12 +- core/pipeline/batch/BatchItem.h | 4 +- core/pipeline/batch/BatchedEvents.h | 2 - core/pipeline/serializer/SLSSerializer.h | 1 - core/plugin/flusher/sls/FlusherSLS.cpp | 1 - core/plugin/input/InputFile.cpp | 2 +- .../processor/ProcessorParseJsonNative.h | 2 +- .../inner/ProcessorSplitLogStringNative.cpp | 18 +- .../inner/ProcessorSplitLogStringNative.h | 1 - ...ProcessorSplitMultilineLogStringNative.cpp | 20 +- .../ProcessorSplitMultilineLogStringNative.h | 1 - .../processor/inner/ProcessorTagNative.cpp | 44 +- .../processor/inner/ProcessorTagNative.h | 9 +- core/runner/ProcessorRunner.cpp | 13 +- core/unittest/batch/BatchItemUnittest.cpp | 3 +- .../file_source/FileTagOptionsUnittest.cpp | 47 +-- core/unittest/flusher/FlusherSLSUnittest.cpp | 67 ++- core/unittest/input/InputFileUnittest.cpp | 4 - .../unittest/metadata/K8sMetadataUnittest.cpp | 4 +- .../models/PipelineEventGroupUnittest.cpp | 27 +- .../pipeline/GlobalConfigUnittest.cpp | 9 +- .../ProcessorSplitLogStringNativeUnittest.cpp | 18 +- .../processor/ProcessorTagNativeUnittest.cpp | 80 ++-- core/unittest/reader/DeletedFileUnittest.cpp | 5 +- core/unittest/reader/FileTagUnittest.cpp | 385 ++++-------------- .../RemoveLastIncompleteLogUnittest.cpp | 57 ++- core/unittest/sender/SenderUnittest.cpp | 66 --- .../serializer/SLSSerializerUnittest.cpp | 17 +- .../serializer/SerializerUnittest.cpp | 1 - docs/cn/configuration/collection-config.md | 6 +- docs/cn/plugins/input/native/input-file.md | 16 +- pkg/config/global_config.go | 1 - pkg/helper/docker_center.go | 17 +- pkg/protocol/converter/converter.go | 38 +- pluginmanager/plugin_runner_helper.go | 14 - pluginmanager/processor_tag_commuinty.go | 50 ++- pluginmanager/processor_tag_community_test.go | 49 +-- 52 files changed, 693 insertions(+), 1074 deletions(-) delete mode 100644 core/common/TagConstants.h mode change 100644 => 100755 core/unittest/file_source/FileTagOptionsUnittest.cpp mode change 100644 => 100755 core/unittest/reader/FileTagUnittest.cpp mode change 100644 => 100755 pluginmanager/processor_tag_community_test.go diff --git a/core/common/CircularBuffer.h b/core/common/CircularBuffer.h index eb16e7869e..a8c3593968 100644 --- a/core/common/CircularBuffer.h +++ b/core/common/CircularBuffer.h @@ -23,7 +23,6 @@ #include "MemoryBarrier.h" #include "Semaphore.h" -#include "common/TimeUtil.h" namespace logtail { diff --git a/core/common/ParamExtractor.cpp b/core/common/ParamExtractor.cpp index 36819ca0e8..b29a0151bc 100644 --- a/core/common/ParamExtractor.cpp +++ b/core/common/ParamExtractor.cpp @@ -192,4 +192,59 @@ bool IsValidMap(const Json::Value& config, const string& key, string& errorMsg) return true; } + +string ParseDefaultAddedTag(const Json::Value* config, + const string& configField, + const string& defaultTagKeyValue, + const PipelineContext& context, + const string& pluginType) { + string errorMsg; + string customTagKey = DEFAULT_CONFIG_TAG_KEY_VALUE; + if (config && config->isMember(configField)) { + if (!GetOptionalStringParam(*config, configField, customTagKey, errorMsg)) { + PARAM_WARNING_DEFAULT(context.GetLogger(), + context.GetAlarm(), + errorMsg, + "__default__", + pluginType, + context.GetConfigName(), + context.GetProjectName(), + context.GetLogstoreName(), + context.GetRegion()); + } + if (customTagKey == DEFAULT_CONFIG_TAG_KEY_VALUE) { + return defaultTagKeyValue; + } + return customTagKey; + } + return defaultTagKeyValue; +} + +string ParseOptionalTag(const Json::Value* config, + const string& configField, + const string& defaultTagKeyValue, + const PipelineContext& context, + const string& pluginType) { + string errorMsg; + string customTagKey; + if (config && config->isMember(configField)) { + if (!GetOptionalStringParam(*config, configField, customTagKey, errorMsg)) { + PARAM_WARNING_DEFAULT(context.GetLogger(), + context.GetAlarm(), + errorMsg, + "__default__", + pluginType, + context.GetConfigName(), + context.GetProjectName(), + context.GetLogstoreName(), + context.GetRegion()); + } + if (customTagKey == DEFAULT_CONFIG_TAG_KEY_VALUE) { + return defaultTagKeyValue; + } + return customTagKey; + } + return ""; +} + } // namespace logtail diff --git a/core/common/ParamExtractor.h b/core/common/ParamExtractor.h index 9122066abd..ef943addce 100644 --- a/core/common/ParamExtractor.h +++ b/core/common/ParamExtractor.h @@ -26,8 +26,10 @@ #include "json/json.h" #include "common/StringTools.h" +#include "constants/TagConstants.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" +#include "pipeline/PipelineContext.h" #define PARAM_ERROR_RETURN(logger, alarm, msg, module, config, project, logstore, region) \ if (module.empty()) { \ @@ -325,6 +327,14 @@ bool IsValidList(const Json::Value& config, const std::string& key, std::string& bool IsValidMap(const Json::Value& config, const std::string& key, std::string& errorMsg); -bool IsKeyExist(const Json::Value& config, const std::string& key); - +std::string ParseDefaultAddedTag(const Json::Value* config, + const std::string& configField, + const std::string& defaultTagKeyValue, + const PipelineContext& context, + const std::string& pluginType); +std::string ParseOptionalTag(const Json::Value* config, + const std::string& configField, + const std::string& defaultTagKeyValue, + const PipelineContext& context, + const std::string& pluginType); } // namespace logtail diff --git a/core/common/TagConstants.h b/core/common/TagConstants.h deleted file mode 100644 index 35f786a11d..0000000000 --- a/core/common/TagConstants.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2024 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -namespace logtail { - -// TODO: move to metadata -const std::string LOG_RESERVED_KEY_PACKAGE_ID = "__pack_id__"; -const std::string LOG_RESERVED_KEY_TRUNCATE_INFO = "__truncate_info__"; - -const std::string DEFAULT_CONFIG_TAG_KEY_VALUE = "__default__"; - -enum TagKey { - FILE_OFFSET_KEY = 0, - FILE_INODE_TAG_KEY = 1, - FILE_PATH_TAG_KEY = 2, - K8S_NAMESPACE_TAG_KEY = 3, - K8S_POD_NAME_TAG_KEY = 4, - K8S_POD_UID_TAG_KEY = 5, - CONTAINER_NAME_TAG_KEY = 6, - CONTAINER_IP_TAG_KEY = 7, - CONTAINER_IMAGE_NAME_TAG_KEY = 8, - K8S_CONTAINER_NAME_TAG_KEY = 9, - K8S_CONTAINER_IMAGE_NAME_TAG_KEY = 10, - K8S_CONTAINER_IP_TAG_KEY = 11, - HOST_NAME = 12, - NUM_VALUES = 13, - UNKOWN = 14, -}; - -#ifdef __ENTERPRISE__ -const std::string TagKeyDefaultValue[NUM_VALUES] = { - "__file_offset__", - "__inode__", - "__path__", - "_namespace_", - "_pod_name_", - "_pod_uid_", - "_container_name_", - "_container_ip_", - "_image_name_", - "_container_name_", - "_image_name_", - "_container_ip_", - "__hostname__", -}; -const std::string AGENT_TAG_DEFAULT_KEY = "__user_defined_id__"; -#else -const std::string TagKeyDefaultValue[NUM_VALUES] = { - "log.file.offset", - "log.file.inode", - "log.file.path", - "k8s.namespace.name", - "k8s.pod.name", - "k8s.pod.uid", - "container.name", - "container.ip", - "container.image.name", - "k8s.container.name", - "k8s.container.image.name", - "k8s.container.ip", - "host.name", -}; -const std::string HOST_IP_DEFAULT_KEY = "host.ip"; -#endif - -} // namespace logtail \ No newline at end of file diff --git a/core/constants/TagConstants.cpp b/core/constants/TagConstants.cpp index cc9edbec1b..68123e43a5 100644 --- a/core/constants/TagConstants.cpp +++ b/core/constants/TagConstants.cpp @@ -16,6 +16,29 @@ namespace logtail { +const std::string& TagKeyToString(TagKey key) { + static const std::vector TagKeyDefaultValue = { + DEFAULT_LOG_TAG_FILE_OFFSET, + DEFAULT_LOG_TAG_FILE_INODE, + DEFAULT_LOG_TAG_FILE_PATH, + DEFAULT_LOG_TAG_NAMESPACE, + DEFAULT_LOG_TAG_POD_NAME, + DEFAULT_LOG_TAG_POD_UID, + DEFAULT_LOG_TAG_CONTAINER_NAME, + DEFAULT_LOG_TAG_CONTAINER_IP, + DEFAULT_LOG_TAG_IMAGE_NAME, + DEFAULT_LOG_TAG_HOST_NAME, + DEFAULT_LOG_TAG_HOST_ID, + DEFAULT_LOG_TAG_CLOUD_PROVIDER, +#ifndef __ENTERPRISE__ + DEFAULT_LOG_TAG_HOST_IP, +#else + DEFAULT_LOG_TAG_USER_DEFINED_ID, +#endif + }; + return TagKeyDefaultValue[key]; +} + ////////////////////////// COMMON //////////////////////// const std::string DEFAULT_TAG_NAMESPACE = "namespace"; const std::string DEFAULT_TAG_HOST_NAME = "host_name"; @@ -26,6 +49,8 @@ const std::string DEFAULT_TAG_CONTAINER_NAME = "container_name"; const std::string DEFAULT_TAG_CONTAINER_IP = "container_ip"; const std::string DEFAULT_TAG_IMAGE_NAME = "image_name"; +const std::string DEFAULT_CONFIG_TAG_KEY_VALUE = "__default__"; + ////////////////////////// LOG //////////////////////// #ifndef __ENTERPRISE__ // 开源版 const std::string DEFAULT_LOG_TAG_HOST_NAME = DEFAULT_TAG_HOST_NAME; @@ -38,8 +63,11 @@ const std::string DEFAULT_LOG_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; const std::string DEFAULT_LOG_TAG_FILE_OFFSET = "file_offset"; const std::string DEFAULT_LOG_TAG_FILE_INODE = "file_inode"; const std::string DEFAULT_LOG_TAG_FILE_PATH = "file_path"; +const std::string DEFAULT_LOG_TAG_TOPIC = "topic"; const std::string DEFAULT_LOG_TAG_HOST_IP = DEFAULT_TAG_HOST_IP; +const std::string DEFAULT_LOG_TAG_HOST_ID = "host_id"; +const std::string DEFAULT_LOG_TAG_CLOUD_PROVIDER = "cloud_provider"; #else const std::string DEFAULT_LOG_TAG_HOST_NAME = "__hostname__"; const std::string DEFAULT_LOG_TAG_NAMESPACE = "_namespace_"; @@ -51,10 +79,18 @@ const std::string DEFAULT_LOG_TAG_IMAGE_NAME = "_image_name_"; const std::string DEFAULT_LOG_TAG_FILE_OFFSET = "__file_offset__"; const std::string DEFAULT_LOG_TAG_FILE_INODE = "__inode__"; const std::string DEFAULT_LOG_TAG_FILE_PATH = "__path__"; +const std::string DEFAULT_LOG_TAG_TOPIC = "__topic__"; +const std::string DEFAULT_LOG_TAG_HOST_ID = "__host_id__"; +const std::string DEFAULT_LOG_TAG_CLOUD_PROVIDER = "__cloud_provider__"; const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID = "__user_defined_id__"; #endif +const std::string LOG_RESERVED_KEY_SOURCE = "__source__"; +const std::string LOG_RESERVED_KEY_TOPIC = "__topic__"; +const std::string LOG_RESERVED_KEY_MACHINE_UUID = "__machine_uuid__"; +const std::string LOG_RESERVED_KEY_PACKAGE_ID = "__pack_id__"; + ////////////////////////// METRIC //////////////////////// const std::string DEFAULT_METRIC_TAG_NAMESPACE = DEFAULT_TAG_NAMESPACE; const std::string DEFAULT_METRIC_TAG_POD_NAME = DEFAULT_TAG_POD_NAME; diff --git a/core/constants/TagConstants.h b/core/constants/TagConstants.h index d57bcd0e26..32d6287e5e 100644 --- a/core/constants/TagConstants.h +++ b/core/constants/TagConstants.h @@ -16,9 +16,35 @@ #pragma once #include +#include namespace logtail { +enum TagKey : int { + FILE_OFFSET_KEY, + FILE_INODE_TAG_KEY, + FILE_PATH_TAG_KEY, + K8S_NAMESPACE_TAG_KEY, + K8S_POD_NAME_TAG_KEY, + K8S_POD_UID_TAG_KEY, + CONTAINER_NAME_TAG_KEY, + CONTAINER_IP_TAG_KEY, + CONTAINER_IMAGE_NAME_TAG_KEY, + HOST_NAME, + HOST_ID, + CLOUD_PROVIDER, +#ifndef __ENTERPRISE__ + HOST_IP, +#else + AGENT_TAG, +#endif +}; + +const std::string& TagKeyToString(TagKey key); + +////////////////////////// COMMON //////////////////////// +extern const std::string DEFAULT_CONFIG_TAG_KEY_VALUE; + ////////////////////////// LOG //////////////////////// extern const std::string DEFAULT_LOG_TAG_HOST_NAME; extern const std::string DEFAULT_LOG_TAG_NAMESPACE; @@ -30,12 +56,20 @@ extern const std::string DEFAULT_LOG_TAG_IMAGE_NAME; extern const std::string DEFAULT_LOG_TAG_FILE_OFFSET; extern const std::string DEFAULT_LOG_TAG_FILE_INODE; extern const std::string DEFAULT_LOG_TAG_FILE_PATH; +extern const std::string DEFAULT_LOG_TAG_TOPIC; +extern const std::string DEFAULT_LOG_TAG_HOST_ID; +extern const std::string DEFAULT_LOG_TAG_CLOUD_PROVIDER; #ifndef __ENTERPRISE__ extern const std::string DEFAULT_LOG_TAG_HOST_IP; #else extern const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID; #endif +extern const std::string LOG_RESERVED_KEY_SOURCE; +extern const std::string LOG_RESERVED_KEY_TOPIC; +extern const std::string LOG_RESERVED_KEY_MACHINE_UUID; +extern const std::string LOG_RESERVED_KEY_PACKAGE_ID; + ////////////////////////// METRIC //////////////////////// extern const std::string DEFAULT_METRIC_TAG_NAMESPACE; extern const std::string DEFAULT_METRIC_TAG_POD_NAME; diff --git a/core/file_server/ContainerInfo.cpp b/core/file_server/ContainerInfo.cpp index d8a3435db2..4bb450abd7 100644 --- a/core/file_server/ContainerInfo.cpp +++ b/core/file_server/ContainerInfo.cpp @@ -14,36 +14,23 @@ #include "file_server/ContainerInfo.h" -#include +#include +#include +#include #include "common/StringTools.h" #include "logger/Logger.h" +#include "models/PipelineEventGroup.h" namespace logtail { -// internal name, used for the communication between C++ and Go -const std::vector containerNameTag = { - "_image_name_", - "_container_name_", - "_pod_name_", - "_namespace_", - "_pod_uid_", - "_container_ip_", - "_k8s_image_name_", - "_k8s_container_name_", - "_k8s_container_ip_", -}; - -const std::vector containerNameTagKey = { - TagKey::CONTAINER_IMAGE_NAME_TAG_KEY, - TagKey::CONTAINER_NAME_TAG_KEY, - TagKey::K8S_POD_NAME_TAG_KEY, - TagKey::K8S_NAMESPACE_TAG_KEY, - TagKey::K8S_POD_UID_TAG_KEY, - TagKey::CONTAINER_IP_TAG_KEY, - TagKey::K8S_CONTAINER_IMAGE_NAME_TAG_KEY, - TagKey::K8S_CONTAINER_NAME_TAG_KEY, - TagKey::K8S_CONTAINER_IP_TAG_KEY, +const std::unordered_map containerNameTag = { + {"_image_name_", TagKey::CONTAINER_IMAGE_NAME_TAG_KEY}, + {"_container_name_", TagKey::CONTAINER_NAME_TAG_KEY}, + {"_pod_name_", TagKey::K8S_POD_NAME_TAG_KEY}, + {"_namespace_", TagKey::K8S_NAMESPACE_TAG_KEY}, + {"_pod_uid_", TagKey::K8S_POD_UID_TAG_KEY}, + {"_container_ip_", TagKey::CONTAINER_IP_TAG_KEY}, }; bool ContainerInfo::ParseAllByJSONObj(const Json::Value& paramsAll, @@ -75,6 +62,7 @@ bool ContainerInfo::ParseAllByJSONObj(const Json::Value& paramsAll, } bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& containerInfo, std::string& errorMsg) { + bool isOldCheckpoint = !params.isMember("MetaDatas"); containerInfo.mJson = params; if (params.isMember("ID") && params["ID"].isString()) { if (params["ID"].empty()) { @@ -110,10 +98,7 @@ bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& con const Json::Value& metaDatas = params["MetaDatas"]; for (Json::ArrayIndex i = 1; i < metaDatas.size(); i += 2) { if (metaDatas[i].isString() && metaDatas[i - 1].isString()) { - sls_logs::LogTag tag; - tag.set_key(metaDatas[i - 1].asString()); - tag.set_value(metaDatas[i].asString()); - containerInfo.mMetadatas->emplace_back(tag); + containerInfo.AddMetadata(metaDatas[i - 1].asString(), metaDatas[i].asString()); } } } @@ -121,10 +106,14 @@ bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& con const Json::Value& tags = params["Tags"]; for (Json::ArrayIndex i = 1; i < tags.size(); i += 2) { if (tags[i].isString() && tags[i - 1].isString()) { - sls_logs::LogTag tag; - tag.set_key(tags[i - 1].asString()); - tag.set_value(tags[i].asString()); - containerInfo.mTags->emplace_back(tag); + std::string key = tags[i - 1].asString(); + std::string value = tags[i].asString(); + // 不是老版本 + if (isOldCheckpoint && containerNameTag.find(key) != containerNameTag.end()) { + containerInfo.AddMetadata(key, value); + } else { + containerInfo.mTags.emplace_back(key, value); + } } } } @@ -137,13 +126,11 @@ bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& con return true; } -TagKey ContainerInfo::GetFileTagKey(const std::string& key) { - for (size_t i = 0; i < containerNameTag.size(); ++i) { - if (containerNameTag[i] == key) { - return containerNameTagKey[i]; - } +void ContainerInfo::AddMetadata(const std::string& key, const std::string& value) { + auto it = containerNameTag.find(key); + if (it != containerNameTag.end()) { + mMetadatas.emplace_back(it->second, value); } - return TagKey::UNKOWN; } } // namespace logtail diff --git a/core/file_server/ContainerInfo.h b/core/file_server/ContainerInfo.h index 5cc1cb588c..c76f7c694f 100644 --- a/core/file_server/ContainerInfo.h +++ b/core/file_server/ContainerInfo.h @@ -20,12 +20,13 @@ #include #include +#include #include #include "json/json.h" -#include "container_manager/ConfigContainerInfoUpdateCmd.h" -#include "protobuf/sls/sls_logs.pb.h" +#include "PipelineEventGroup.h" +#include "constants/TagConstants.h" namespace logtail { @@ -45,19 +46,12 @@ struct ContainerInfo { std::string mLogPath; std::string mUpperDir; std::vector mMounts; // mounts of this container - // Tags is hold by each reader and cost memory, so use shared_ptr to share the same tags. - std::shared_ptr> mTags; // ContainerNameTag, ExternalEnvTag and ExternalK8sLabelTag. - std::shared_ptr> mMetadatas; + std::vector> mTags; // ExternalEnvTag and ExternalK8sLabelTag. + std::vector> mMetadatas; // ContainerNameTag Json::Value mJson; // this obj's json, for saving to local file - ContainerInfo() { - mTags = std::make_shared>(); - mMetadatas = std::make_shared>(); - } - static bool ParseByJSONObj(const Json::Value&, ContainerInfo&, std::string&); static bool ParseAllByJSONObj(const Json::Value&, std::unordered_map&, std::string&); - static TagKey GetFileTagKey(const std::string& key); bool operator==(const ContainerInfo& rhs) const { if (mID != rhs.mID) { @@ -82,23 +76,23 @@ struct ContainerInfo { return false; } } - if (mMetadatas->size() != rhs.mMetadatas->size()) { + if (mMetadatas.size() != rhs.mMetadatas.size()) { return false; } - for (size_t idx = 0; idx < mMetadatas->size(); ++idx) { - const auto& lhsTag = (*mMetadatas)[idx]; - const auto& rhsTag = (*rhs.mMetadatas)[idx]; - if (lhsTag.key() != rhsTag.key() || lhsTag.value() != rhsTag.value()) { + for (size_t idx = 0; idx < mMetadatas.size(); ++idx) { + const auto& lhsTag = mMetadatas[idx]; + const auto& rhsTag = rhs.mMetadatas[idx]; + if (lhsTag.first != rhsTag.first || lhsTag.second != rhsTag.second) { return false; } } - if (mTags->size() != rhs.mTags->size()) { + if (mTags.size() != rhs.mTags.size()) { return false; } - for (size_t idx = 0; idx < mTags->size(); ++idx) { - const auto& lhsTag = (*mTags)[idx]; - const auto& rhsTag = (*rhs.mTags)[idx]; - if (lhsTag.key() != rhsTag.key() || lhsTag.value() != rhsTag.value()) { + for (size_t idx = 0; idx < mTags.size(); ++idx) { + const auto& lhsTag = mTags[idx]; + const auto& rhsTag = rhs.mTags[idx]; + if (lhsTag.first != rhsTag.first || lhsTag.second != rhsTag.second) { return false; } } @@ -106,6 +100,8 @@ struct ContainerInfo { } bool operator!=(const ContainerInfo& rhs) const { return !(*this == rhs); } + void AddMetadata(const std::string& key, const std::string& value); + private: }; diff --git a/core/file_server/FileTagOptions.cpp b/core/file_server/FileTagOptions.cpp index 72d1fa34f3..a285313a6f 100644 --- a/core/file_server/FileTagOptions.cpp +++ b/core/file_server/FileTagOptions.cpp @@ -16,16 +16,23 @@ #include "file_server/FileTagOptions.h" +#include + +#include "PipelineContext.h" +#include "TagConstants.h" #include "common/ParamExtractor.h" +using namespace std; + namespace logtail { bool FileTagOptions::Init(const Json::Value& config, const PipelineContext& context, - const std::string& pluginType, + const string& pluginType, bool enableContainerDiscovery) { - std::string errorMsg; + string errorMsg; + // Deprecated: should use FileInodeTagKey instead // AppendingLogPositionMeta bool appendingLogPositionMeta = false; if (!GetOptionalBoolParam(config, "AppendingLogPositionMeta", appendingLogPositionMeta, errorMsg)) { @@ -45,44 +52,42 @@ bool FileTagOptions::Init(const Json::Value& config, const Json::Value* tagConfig = config.find(tagKey, tagKey + strlen(tagKey)); if (tagConfig) { if (!tagConfig->isObject()) { - PARAM_WARNING_DEFAULT(context.GetLogger(), - context.GetAlarm(), - "param Tags is not of type object", - "custom", - pluginType, - context.GetConfigName(), - context.GetProjectName(), - context.GetLogstoreName(), - context.GetRegion()); + PARAM_WARNING_IGNORE(context.GetLogger(), + context.GetAlarm(), + "param Tags is not of type object", + pluginType, + context.GetConfigName(), + context.GetProjectName(), + context.GetLogstoreName(), + context.GetRegion()); tagConfig = nullptr; } } // the priority of FileOffsetKey and FileInodeTagKey is higher than appendingLogPositionMeta if (config.isMember("FileOffsetKey") || (tagConfig && tagConfig->isMember("FileInodeTagKey"))) { - parseDefaultNotAddTag(&config, "FileOffsetKey", TagKey::FILE_OFFSET_KEY, context, pluginType); - parseDefaultNotAddTag(tagConfig, "FileInodeTagKey", TagKey::FILE_INODE_TAG_KEY, context, pluginType); + parseTagKey(&config, "FileOffsetKey", TagKey::FILE_OFFSET_KEY, context, pluginType, false); + parseTagKey(tagConfig, "FileInodeTagKey", TagKey::FILE_INODE_TAG_KEY, context, pluginType, false); } else if (appendingLogPositionMeta) { - mFileTags[TagKey::FILE_OFFSET_KEY] = TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]; - mFileTags[TagKey::FILE_INODE_TAG_KEY] = TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]; + mFileTags[TagKey::FILE_OFFSET_KEY] = TagKeyToString(TagKey::FILE_OFFSET_KEY); + mFileTags[TagKey::FILE_INODE_TAG_KEY] = TagKeyToString(TagKey::FILE_INODE_TAG_KEY); } - parseDefaultAddTag(tagConfig, "FilePathTagKey", TagKey::FILE_PATH_TAG_KEY, context, pluginType); + auto filePathTagKey = ParseDefaultAddedTag( + tagConfig, "FilePathTagKey", TagKeyToString(TagKey::FILE_PATH_TAG_KEY), context, pluginType); + if (!filePathTagKey.empty()) { + mFileTags[TagKey::FILE_PATH_TAG_KEY] = filePathTagKey; + } // ContainerDiscovery if (enableContainerDiscovery) { - parseDefaultAddTag(tagConfig, "K8sNamespaceTagKey", TagKey::K8S_NAMESPACE_TAG_KEY, context, pluginType); - parseDefaultAddTag(tagConfig, "K8sPodNameTagKey", TagKey::K8S_POD_NAME_TAG_KEY, context, pluginType); - parseDefaultAddTag(tagConfig, "K8sPodUidTagKey", TagKey::K8S_POD_UID_TAG_KEY, context, pluginType); - parseDefaultAddTag(tagConfig, "ContainerNameTagKey", TagKey::K8S_CONTAINER_NAME_TAG_KEY, context, pluginType); - parseDefaultAddTag(tagConfig, "ContainerIpTagKey", TagKey::K8S_CONTAINER_IP_TAG_KEY, context, pluginType); - parseDefaultAddTag( - tagConfig, "ContainerImageNameTagKey", TagKey::K8S_CONTAINER_IMAGE_NAME_TAG_KEY, context, pluginType); - - parseDefaultAddTag(tagConfig, "ContainerNameTagKey", TagKey::CONTAINER_NAME_TAG_KEY, context, pluginType); - parseDefaultAddTag(tagConfig, "ContainerIpTagKey", TagKey::CONTAINER_IP_TAG_KEY, context, pluginType); - parseDefaultAddTag( - tagConfig, "ContainerImageNameTagKey", TagKey::CONTAINER_IMAGE_NAME_TAG_KEY, context, pluginType); + parseTagKey(tagConfig, "K8sNamespaceTagKey", TagKey::K8S_NAMESPACE_TAG_KEY, context, pluginType, true); + parseTagKey(tagConfig, "K8sPodNameTagKey", TagKey::K8S_POD_NAME_TAG_KEY, context, pluginType, true); + parseTagKey(tagConfig, "K8sPodUidTagKey", TagKey::K8S_POD_UID_TAG_KEY, context, pluginType, true); + parseTagKey(tagConfig, "ContainerNameTagKey", TagKey::CONTAINER_NAME_TAG_KEY, context, pluginType, true); + parseTagKey(tagConfig, "ContainerIpTagKey", TagKey::CONTAINER_IP_TAG_KEY, context, pluginType, true); + parseTagKey( + tagConfig, "ContainerImageNameTagKey", TagKey::CONTAINER_IMAGE_NAME_TAG_KEY, context, pluginType, true); } return true; @@ -97,62 +102,25 @@ StringView FileTagOptions::GetFileTagKeyName(TagKey key) const { return StringView(); } -void FileTagOptions::parseDefaultAddTag(const Json::Value* config, - const std::string& keyName, - const TagKey& keyEnum, - const PipelineContext& context, - const std::string& pluginType) { - std::string errorMsg; - std::string key; - if (config && config->isMember(keyName)) { - if (!GetOptionalStringParam(*config, keyName, key, errorMsg)) { - PARAM_WARNING_DEFAULT(context.GetLogger(), - context.GetAlarm(), - errorMsg, - "custom", - pluginType, - context.GetConfigName(), - context.GetProjectName(), - context.GetLogstoreName(), - context.GetRegion()); - } else if (!key.empty()) { - if (key == DEFAULT_CONFIG_TAG_KEY_VALUE) { - mFileTags[keyEnum] = TagKeyDefaultValue[keyEnum]; - } else { - mFileTags[keyEnum] = key; - } - } - } else { - mFileTags[keyEnum] = TagKeyDefaultValue[keyEnum]; - } +bool FileTagOptions::IsEnableLogPositionMeta() { + return !mFileTags[TagKey::FILE_OFFSET_KEY].empty() || !mFileTags[TagKey::FILE_INODE_TAG_KEY].empty(); } -void FileTagOptions::parseDefaultNotAddTag(const Json::Value* config, - const std::string& keyName, - const TagKey& keyEnum, - const PipelineContext& context, - const std::string& pluginType) { - std::string errorMsg; - std::string key; - if (config && config->isMember(keyName)) { - if (!GetOptionalStringParam(*config, keyName, key, errorMsg)) { - PARAM_WARNING_DEFAULT(context.GetLogger(), - context.GetAlarm(), - errorMsg, - "custom", - pluginType, - context.GetConfigName(), - context.GetProjectName(), - context.GetLogstoreName(), - context.GetRegion()); - } else if (!key.empty()) { - if (key == DEFAULT_CONFIG_TAG_KEY_VALUE) { - mFileTags[keyEnum] = TagKeyDefaultValue[keyEnum]; - } else { - mFileTags[keyEnum] = key; - } - } +void FileTagOptions::parseTagKey(const Json::Value* config, + const string& configField, + TagKey tagKey, + const PipelineContext& context, + const std::string& pluginType, + bool defaultAdded) { + string customTagKey; + if (defaultAdded) { + customTagKey = ParseDefaultAddedTag(config, configField, TagKeyToString(tagKey), context, pluginType); + } else { + customTagKey = ParseOptionalTag(config, configField, TagKeyToString(tagKey), context, pluginType); + } + if (!customTagKey.empty()) { + mFileTags[tagKey] = customTagKey; } } -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/file_server/FileTagOptions.h b/core/file_server/FileTagOptions.h index 931f8fc2cf..6a8f2ced6a 100644 --- a/core/file_server/FileTagOptions.h +++ b/core/file_server/FileTagOptions.h @@ -18,9 +18,10 @@ #include +#include #include -#include "common/TagConstants.h" +#include "constants/TagConstants.h" #include "pipeline/PipelineContext.h" namespace logtail { @@ -32,20 +33,15 @@ class FileTagOptions { const std::string& pluginType, bool enableContainerDiscovery); StringView GetFileTagKeyName(TagKey key) const; - + bool IsEnableLogPositionMeta(); private: - void parseDefaultAddTag(const Json::Value* config, - const std::string& keyName, - const TagKey& keyEnum, - const PipelineContext& context, - const std::string& pluginType); - void parseDefaultNotAddTag(const Json::Value* config, - const std::string& keyName, - const TagKey& keyEnum, - const PipelineContext& context, - const std::string& pluginType); - + void parseTagKey(const Json::Value* config, + const std::string& configField, + TagKey tagKey, + const PipelineContext& context, + const std::string& pluginType, + bool defaultAdded); std::unordered_map mFileTags; #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 5a88529341..3a67070305 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -14,6 +14,10 @@ #include "file_server/reader/LogFileReader.h" +#include "Monitor.h" +#include "PipelineEventGroup.h" +#include "TagConstants.h" + #if defined(_MSC_VER) #include #include @@ -125,7 +129,8 @@ LogFileReader* LogFileReader::CreateLogFileReader(const string& hostLogPathDir, ? discoveryConfig.first->GetWildcardPaths()[0] : discoveryConfig.first->GetBasePath(), containerPath->mRealBaseDir.size()); - reader->SetContainerExtraTags(containerPath->mTags); + reader->AddContainerMetadatas(containerPath->mMetadatas); + reader->AddExtraTags(containerPath->mTags); } } @@ -773,10 +778,7 @@ std::string LogFileReader::GetTopicName(const std::string& topicConfig, const st if (matchedSize == (size_t)1) { // != default topic name if (keys[0] != "__topic_1__") { - sls_logs::LogTag tag; - tag.set_key(keys[0]); - tag.set_value(values[0]); - mTopicExtraTags.push_back(tag); + mTopicExtraTags.emplace_back(keys[0], values[0]); } return values[0]; } else { @@ -786,10 +788,7 @@ std::string LogFileReader::GetTopicName(const std::string& topicConfig, const st } else { res = res + "_" + values[i]; } - sls_logs::LogTag tag; - tag.set_key(keys[i]); - tag.set_value(values[i]); - mTopicExtraTags.push_back(tag); + mTopicExtraTags.emplace_back(keys[i], values[i]); } } return res; @@ -811,10 +810,7 @@ std::string LogFileReader::GetTopicName(const std::string& topicConfig, const st res = res + "_" + what[i]; } if (matchedSize > 2) { - sls_logs::LogTag tag; - tag.set_key(string("__topic_") + ToString(i) + "__"); - tag.set_value(what[i]); - mTopicExtraTags.push_back(tag); + mTopicExtraTags.emplace_back(string("__topic_") + ToString(i) + "__", what[i]); } } } else { @@ -2470,62 +2466,48 @@ void LogFileReader::SetEventGroupMetaAndTag(PipelineEventGroup& group) { } bool isContainerLog = mFileLogFormat == LogFormat::DOCKER_JSON_FILE || mFileLogFormat == LogFormat::CONTAINERD_TEXT; if (!isContainerLog) { - group.SetMetadata(EventGroupMetaKey::LOG_FILE_PATH, GetConvertedPath()); group.SetMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, GetHostLogPath()); - group.SetMetadata(EventGroupMetaKey::LOG_FILE_INODE, ToString(GetDevInode().inode)); } group.SetMetadata(EventGroupMetaKey::SOURCE_ID, GetSourceId()); - group.SetMetadata(EventGroupMetaKey::SOURCE, LogFileProfiler::mIpAddr); - group.SetMetadata(EventGroupMetaKey::TOPIC, GetTopicName()); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, Application::GetInstance()->GetUUID()); + // we store info which users can see in tags + // for log, these includes: + // 1. topic + group.SetTag(DEFAULT_LOG_TAG_TOPIC, GetTopicName()); + auto topicExtraTags = GetTopicExtraTags(); + for (auto& tag : topicExtraTags) { + group.SetTag(tag.first, tag.second); + } + // 2. external k8s env/label tag + auto extraTags = GetExtraTags(); + for (auto& tag : extraTags) { + group.SetTag(tag.first, tag.second); + } + // 3. offset, inode, path, container name tag if (mTagConfig.first != nullptr) { auto offsetKey = mTagConfig.first->GetFileTagKeyName(TagKey::FILE_OFFSET_KEY); if (!offsetKey.empty()) { group.SetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY, offsetKey); } - } - // we store info which users can see in tags - // for log, these includes: - // 1. extra topic - auto topicExtraTags = GetTopicExtraTags(); - for (size_t i = 0; i < topicExtraTags.size(); ++i) { - group.SetTag(topicExtraTags[i].key(), topicExtraTags[i].value()); - } - // 2. container name tag, external k8s env/label tag - auto containerExtraTags = GetContainerExtraTags(); - if (containerExtraTags) { - for (size_t i = 0; i < containerExtraTags->size(); ++i) { - auto key = ContainerInfo::GetFileTagKey((*containerExtraTags)[i].key()); - if (key != TagKey::UNKOWN) { // container name tag - StringBuffer b = group.GetSourceBuffer()->CopyString((*containerExtraTags)[i].value()); - if (mTagConfig.first == nullptr) { // no tag config - group.SetTagNoCopy(TagKeyDefaultValue[key], StringView(b.data, b.size)); - } else { - if (mTagConfig.first != nullptr) { - auto keyName = mTagConfig.first->GetFileTagKeyName(key); - if (!keyName.empty()) { - group.SetTagNoCopy(keyName, StringView(b.data, b.size)); - } - } - } - } else { // external k8s env/label tag - group.SetTag((*containerExtraTags)[i].key(), (*containerExtraTags)[i].value()); + auto inodeKey = mTagConfig.first->GetFileTagKeyName(TagKey::FILE_INODE_TAG_KEY); + if (!inodeKey.empty()) { + group.SetTag(inodeKey, ToString(GetDevInode().inode)); + } + + auto pathKey = mTagConfig.first->GetFileTagKeyName(TagKey::FILE_PATH_TAG_KEY); + if (!pathKey.empty()) { + const string& path = GetConvertedPath(); + if (!path.empty()) { + group.SetTagNoCopy(pathKey, path.substr(0, 511)); } } - } - if (mTagConfig.first != nullptr) { - // 3. inode - auto keyName = mTagConfig.first->GetFileTagKeyName(TagKey::FILE_INODE_TAG_KEY); - if (!keyName.empty()) { - StringBuffer b = group.GetSourceBuffer()->CopyString(ToString(GetDevInode().inode)); - group.SetTagNoCopy(keyName, StringView(b.data, b.size)); - } - // 4. path - keyName = mTagConfig.first->GetFileTagKeyName(TagKey::FILE_PATH_TAG_KEY); - if (!keyName.empty()) { - StringBuffer b = group.GetSourceBuffer()->CopyString(GetConvertedPath()); - group.SetTagNoCopy(keyName, StringView(b.data, b.size)); + + auto containerMetadatas = GetContainerMetadatas(); + for (auto& metadata : containerMetadatas) { + auto key = mTagConfig.first->GetFileTagKeyName(metadata.first); + if (!key.empty()) { + group.SetTagNoCopy(key, metadata.second); + } } } } diff --git a/core/file_server/reader/LogFileReader.h b/core/file_server/reader/LogFileReader.h index f9539df215..ce6f8ef099 100644 --- a/core/file_server/reader/LogFileReader.h +++ b/core/file_server/reader/LogFileReader.h @@ -19,10 +19,12 @@ #include #include #include +#include #include #include #include +#include "TagConstants.h" #include "checkpoint/RangeCheckpoint.h" #include "common/DevInode.h" #include "common/EncodingConverter.h" @@ -395,14 +397,18 @@ class LogFileReader { void SetDockerPath(const std::string& dockerBasePath, size_t dockerReplaceSize); - const std::vector& GetTopicExtraTags() const { return mTopicExtraTags; } + const std::vector>& GetTopicExtraTags() const { return mTopicExtraTags; } - void SetTopicExtraTags(const std::vector& tags) { mTopicExtraTags = tags; } + const std::vector>& GetContainerMetadatas() { return mContainerMetadatas; } - const std::shared_ptr>& GetContainerExtraTags() { return mContainerExtraTags; } + void AddContainerMetadatas(const std::vector>& tags) { + mContainerMetadatas.insert(mContainerMetadatas.end(), tags.begin(), tags.end()); + } + + const std::vector>& GetExtraTags() { return mExtraTags; } - void SetContainerExtraTags(const std::shared_ptr> tags) { - mContainerExtraTags = tags; + void AddExtraTags(const std::vector>& tags) { + mExtraTags.insert(mExtraTags.end(), tags.begin(), tags.end()); } QueueKey GetQueueKey() const { return mReaderConfig.second->GetProcessQueueKey(); } @@ -535,8 +541,9 @@ class LogFileReader { std::string mDockerPath; // tags - std::vector mTopicExtraTags; - std::shared_ptr> mContainerExtraTags; + std::vector> mTopicExtraTags; + std::vector> mContainerMetadatas; + std::vector> mExtraTags; // int32_t mCloseUnusedInterval; // PreciseTimestampConfig mPreciseTimestampConfig; diff --git a/core/models/PipelineEventGroup.cpp b/core/models/PipelineEventGroup.cpp index 40bb3f7bae..de86bd15b5 100644 --- a/core/models/PipelineEventGroup.cpp +++ b/core/models/PipelineEventGroup.cpp @@ -326,7 +326,6 @@ bool PipelineEventGroup::IsReplay() const { } #ifdef APSARA_UNIT_TEST_MAIN -const string EVENT_GROUP_META_LOG_FILE_PATH = "log.file.path"; const string EVENT_GROUP_META_LOG_FILE_PATH_RESOLVED = "log.file.path_resolved"; const string EVENT_GROUP_META_LOG_FILE_INODE = "log.file.inode"; const string EVENT_GROUP_META_CONTAINER_TYPE = "container.type"; @@ -350,12 +349,8 @@ const string EVENT_GROUP_META_SOURCE_ID = "source.id"; const string& EventGroupMetaKeyToString(EventGroupMetaKey key) { switch (key) { - case EventGroupMetaKey::LOG_FILE_PATH: - return EVENT_GROUP_META_LOG_FILE_PATH; case EventGroupMetaKey::LOG_FILE_PATH_RESOLVED: return EVENT_GROUP_META_LOG_FILE_PATH_RESOLVED; - case EventGroupMetaKey::LOG_FILE_INODE: - return EVENT_GROUP_META_LOG_FILE_INODE; case EventGroupMetaKey::SOURCE_ID: return EVENT_GROUP_META_SOURCE_ID; case EventGroupMetaKey::LOG_FORMAT: @@ -381,9 +376,7 @@ const string EventGroupMetaValueToString(string value) { EventGroupMetaKey StringToEventGroupMetaKey(const string& key) { static unordered_map sStringToEnum{ - {EVENT_GROUP_META_LOG_FILE_PATH, EventGroupMetaKey::LOG_FILE_PATH}, {EVENT_GROUP_META_LOG_FILE_PATH_RESOLVED, EventGroupMetaKey::LOG_FILE_PATH_RESOLVED}, - {EVENT_GROUP_META_LOG_FILE_INODE, EventGroupMetaKey::LOG_FILE_INODE}, {EVENT_GROUP_META_SOURCE_ID, EventGroupMetaKey::SOURCE_ID}, {EVENT_GROUP_META_HAS_PART_LOG, EventGroupMetaKey::HAS_PART_LOG}}; auto it = sStringToEnum.find(key); diff --git a/core/models/PipelineEventGroup.h b/core/models/PipelineEventGroup.h index a0f1e43ddf..929c6e1ebc 100644 --- a/core/models/PipelineEventGroup.h +++ b/core/models/PipelineEventGroup.h @@ -35,9 +35,7 @@ class EventPool; // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/resource/container.md enum class EventGroupMetaKey { UNKNOWN, - LOG_FILE_PATH, LOG_FILE_PATH_RESOLVED, - LOG_FILE_INODE, LOG_FORMAT, LOG_FILE_OFFSET_KEY, HAS_PART_LOG, @@ -62,10 +60,7 @@ enum class EventGroupMetaKey { PROMETHEUS_STREAM_ID, PROMETHEUS_STREAM_TOTAL, - SOURCE_ID, - TOPIC, - SOURCE, - MACHINE_UUID, + SOURCE_ID }; using GroupMetadata = std::map; diff --git a/core/pipeline/GlobalConfig.cpp b/core/pipeline/GlobalConfig.cpp index b7e52ee25a..edf1c0311a 100644 --- a/core/pipeline/GlobalConfig.cpp +++ b/core/pipeline/GlobalConfig.cpp @@ -14,8 +14,12 @@ #include "pipeline/GlobalConfig.h" +#include +#include + #include "json/json.h" +#include "TagConstants.h" #include "common/ParamExtractor.h" #include "pipeline/PipelineContext.h" #include "pipeline/queue/ProcessQueueManager.h" @@ -24,8 +28,13 @@ using namespace std; namespace logtail { -const unordered_set GlobalConfig::sNativeParam - = {"TopicType", "TopicFormat", "Priority", "EnableTimestampNanosecond", "UsingOldContentTag"}; +const unordered_set GlobalConfig::sNativeParam = {"TopicType", + "TopicFormat", + "Priority", + "EnableTimestampNanosecond", + "UsingOldContentTag", + "PipelineMetaTagKey", + "AgentEnvMetaTagKey"}; bool GlobalConfig::Init(const Json::Value& config, const PipelineContext& ctx, Json::Value& extendedParams) { const string moduleName = "global"; @@ -153,7 +162,8 @@ bool GlobalConfig::Init(const Json::Value& config, const PipelineContext& ctx, J } // PipelineMetaTagKey - if (!GetOptionalMapParam(config, "PipelineMetaTagKey", mPipelineMetaTagKey, errorMsg)) { + unordered_map tagKeys; + if (!GetOptionalMapParam(config, "PipelineMetaTagKey", tagKeys, errorMsg)) { PARAM_WARNING_IGNORE(ctx.GetLogger(), ctx.GetAlarm(), errorMsg, @@ -163,6 +173,27 @@ bool GlobalConfig::Init(const Json::Value& config, const PipelineContext& ctx, J ctx.GetLogstoreName(), ctx.GetRegion()); } +#ifdef __ENTERPRISE__ + mPipelineMetaTagKey[TagKey::AGENT_TAG] = DEFAULT_CONFIG_TAG_KEY_VALUE; +#endif + for (const auto& kv : tagKeys) { + if (kv.first == "HOST_NAME") { + mPipelineMetaTagKey[TagKey::HOST_NAME] = kv.second; + } else if (kv.first == "HOST_ID") { + mPipelineMetaTagKey[TagKey::HOST_ID] = kv.second; + } else if (kv.first == "CLOUD_PROVIDER") { + mPipelineMetaTagKey[TagKey::CLOUD_PROVIDER] = kv.second; + } +#ifdef __ENTERPRISE__ + else if (kv.first == "AGENT_TAG") { + mPipelineMetaTagKey[TagKey::AGENT_TAG] = kv.second; + } +#else + else if (kv.first == "HOST_IP") { + mPipelineMetaTagKey[TagKey::HOST_IP] = kv.second; + } +#endif + } #ifdef __ENTERPRISE__ // AgentEnvMetaTagKey @@ -185,4 +216,12 @@ bool GlobalConfig::Init(const Json::Value& config, const PipelineContext& ctx, J return true; } +Json::Value GlobalConfig::GetPipelineMetaTagKeyJsonValue() const { + Json::Value json; + for (const auto& kv : mPipelineMetaTagKey) { + json[kv.first] = kv.second; + } + return json; +} + } // namespace logtail diff --git a/core/pipeline/GlobalConfig.h b/core/pipeline/GlobalConfig.h index a6e65fdf3a..9184953f74 100644 --- a/core/pipeline/GlobalConfig.h +++ b/core/pipeline/GlobalConfig.h @@ -24,6 +24,8 @@ #include "json/json.h" +#include "constants/TagConstants.h" + namespace logtail { class PipelineContext; @@ -40,14 +42,8 @@ struct GlobalConfig { uint32_t mPriority = 1U; bool mEnableTimestampNanosecond = false; bool mUsingOldContentTag = false; - std::unordered_map mPipelineMetaTagKey; - Json::Value GetPipelineMetaTagKeyJsonValue() const { - Json::Value json; - for (const auto& kv : mPipelineMetaTagKey) { - json[kv.first] = kv.second; - } - return json; - } + std::unordered_map mPipelineMetaTagKey; + Json::Value GetPipelineMetaTagKeyJsonValue() const; #ifdef __ENTERPRISE__ bool mEnableAgentEnvMetaTagControl = false; diff --git a/core/pipeline/batch/BatchItem.h b/core/pipeline/batch/BatchItem.h index d2fb43201a..2f8730773a 100644 --- a/core/pipeline/batch/BatchItem.h +++ b/core/pipeline/batch/BatchItem.h @@ -137,14 +137,12 @@ class EventBatchItem { Clear(); } - void Reset(const GroupMetadata& metadata, - const SizedMap& tags, + void Reset(const SizedMap& tags, const std::shared_ptr& sourceBuffer, const RangeCheckpointPtr& exactlyOnceCheckpoint, StringView packIdPrefix) { Clear(); // should be copied instead of moved in case of one original group splitted into two - mBatch.mMetadata = metadata; mBatch.mTags = tags; mBatch.mExactlyOnceCheckpoint = exactlyOnceCheckpoint; mBatch.mPackIdPrefix = packIdPrefix; diff --git a/core/pipeline/batch/BatchedEvents.h b/core/pipeline/batch/BatchedEvents.h index 239a7319b7..6652e955ec 100644 --- a/core/pipeline/batch/BatchedEvents.h +++ b/core/pipeline/batch/BatchedEvents.h @@ -26,7 +26,6 @@ namespace logtail { struct BatchedEvents { EventsContainer mEvents; - GroupMetadata mMetadata; SizedMap mTags; std::vector> mSourceBuffers; size_t mSizeBytes = 0; // only set on completion @@ -41,7 +40,6 @@ struct BatchedEvents { // for flusher_sls only BatchedEvents(EventsContainer&& events, - GroupMetadata metadata, SizedMap&& tags, std::shared_ptr&& sourceBuffer, StringView packIdPrefix, diff --git a/core/pipeline/serializer/SLSSerializer.h b/core/pipeline/serializer/SLSSerializer.h index 2184862e99..0b92c7134b 100644 --- a/core/pipeline/serializer/SLSSerializer.h +++ b/core/pipeline/serializer/SLSSerializer.h @@ -19,7 +19,6 @@ #include #include -#include "common/TagConstants.h" #include "pipeline/serializer/Serializer.h" namespace logtail { diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 8700fccfb2..b5b91c6704 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -962,7 +962,6 @@ void FlusherSLS::GenerateGoPlugin(const Json::Value& config, Json::Value& res) c bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { string serializedData, compressedData; BatchedEvents g(std::move(group.MutableEvents()), - std::move(group.GetAllMetadata()), std::move(group.GetSizedTags()), std::move(group.GetSourceBuffer()), group.GetMetadata(EventGroupMetaKey::SOURCE_ID), diff --git a/core/plugin/input/InputFile.cpp b/core/plugin/input/InputFile.cpp index 958c3be021..b84c943199 100644 --- a/core/plugin/input/InputFile.cpp +++ b/core/plugin/input/InputFile.cpp @@ -233,7 +233,7 @@ bool InputFile::CreateInnerProcessors() { } detail["EnableRawContent"] = Json::Value(!mContext->HasNativeProcessors() && !mContext->IsExactlyOnceEnabled() - && !mContext->IsFlushingThroughGoPipeline() && !mFileReader.mAppendingLogPositionMeta); + && !mContext->IsFlushingThroughGoPipeline() && !mFileTag.IsEnableLogPositionMeta()); if (!processor->Init(detail, *mContext)) { // should not happen return false; diff --git a/core/plugin/processor/ProcessorParseJsonNative.h b/core/plugin/processor/ProcessorParseJsonNative.h index 214d4de23d..e7edac4fda 100644 --- a/core/plugin/processor/ProcessorParseJsonNative.h +++ b/core/plugin/processor/ProcessorParseJsonNative.h @@ -42,7 +42,7 @@ class ProcessorParseJsonNative : public Processor { PipelineEventPtr& e, bool& sourceKeyOverwritten); void AddLog(const StringView& key, const StringView& value, LogEvent& targetEvent, bool overwritten = true); - bool ProcessEvent(const StringView& logPath, PipelineEventPtr& e); + bool ProcessEvent(const StringView& logPath, PipelineEventPtr& e, const GroupMetadata& metadata); CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; diff --git a/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp b/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp index 681af4cfce..1602e7c6f2 100644 --- a/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp +++ b/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp @@ -55,19 +55,6 @@ bool ProcessorSplitLogStringNative::Init(const Json::Value& config) { mSplitChar = static_cast(splitter); } - // AppendingLogPositionMeta - if (!GetOptionalBoolParam(config, "AppendingLogPositionMeta", mAppendingLogPositionMeta, errorMsg)) { - PARAM_WARNING_DEFAULT(mContext->GetLogger(), - mContext->GetAlarm(), - errorMsg, - mAppendingLogPositionMeta, - sName, - mContext->GetConfigName(), - mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion()); - } - // EnableRawContent if (!GetOptionalBoolParam(config, "EnableRawContent", mEnableRawContent, errorMsg)) { PARAM_WARNING_DEFAULT(mContext->GetLogger(), @@ -160,9 +147,10 @@ void ProcessorSplitLogStringNative::ProcessEvent(PipelineEventGroup& logGroup, ? sourceEvent.GetPosition().second - (content.data() - sourceVal.data()) : content.size() + 1; targetEvent->SetPosition(offset, length); - if (mAppendingLogPositionMeta) { + if (logGroup.HasMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY)) { StringBuffer offsetStr = logGroup.GetSourceBuffer()->CopyString(ToString(offset)); - targetEvent->SetContentNoCopy(LOG_RESERVED_KEY_FILE_OFFSET, StringView(offsetStr.data, offsetStr.size)); + targetEvent->SetContentNoCopy(logGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), + StringView(offsetStr.data, offsetStr.size)); } newEvents.emplace_back(std::move(targetEvent), true, nullptr); } diff --git a/core/plugin/processor/inner/ProcessorSplitLogStringNative.h b/core/plugin/processor/inner/ProcessorSplitLogStringNative.h index e08a4d2ff8..ecc06f6b33 100644 --- a/core/plugin/processor/inner/ProcessorSplitLogStringNative.h +++ b/core/plugin/processor/inner/ProcessorSplitLogStringNative.h @@ -31,7 +31,6 @@ class ProcessorSplitLogStringNative : public Processor { std::string mSourceKey = DEFAULT_CONTENT_KEY; char mSplitChar = '\n'; - bool mAppendingLogPositionMeta = false; bool mEnableRawContent = false; const std::string& Name() const override { return sName; } diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp index 8f9f38217a..2e724e8dfa 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp @@ -20,6 +20,8 @@ #include "boost/regex.hpp" +#include "PipelineEventGroup.h" +#include "TagConstants.h" #include "app_config/AppConfig.h" #include "common/ParamExtractor.h" #include "constants/Constants.h" @@ -52,19 +54,6 @@ bool ProcessorSplitMultilineLogStringNative::Init(const Json::Value& config) { return false; } - // AppendingLogPositionMeta - if (!GetOptionalBoolParam(config, "AppendingLogPositionMeta", mAppendingLogPositionMeta, errorMsg)) { - PARAM_WARNING_DEFAULT(mContext->GetLogger(), - mContext->GetAlarm(), - errorMsg, - mAppendingLogPositionMeta, - sName, - mContext->GetConfigName(), - mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion()); - } - // EnableRawContent if (!GetOptionalBoolParam(config, "EnableRawContent", mEnableRawContent, errorMsg)) { PARAM_WARNING_DEFAULT(mContext->GetLogger(), @@ -330,9 +319,10 @@ void ProcessorSplitMultilineLogStringNative::CreateNewEvent(const StringView& co auto const length = isLastLog ? sourceEvent.GetPosition().second - (content.data() - sourceVal.data()) : content.size() + 1; targetEvent->SetPosition(offset, length); - if (mAppendingLogPositionMeta) { + if (logGroup.HasMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY)) { StringBuffer offsetStr = logGroup.GetSourceBuffer()->CopyString(ToString(offset)); - targetEvent->SetContentNoCopy(LOG_RESERVED_KEY_FILE_OFFSET, StringView(offsetStr.data, offsetStr.size)); + targetEvent->SetContentNoCopy(logGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), + StringView(offsetStr.data, offsetStr.size)); } newEvents.emplace_back(std::move(targetEvent), true, nullptr); } diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h index 0c74b34d1a..4830199ae4 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h @@ -33,7 +33,6 @@ class ProcessorSplitMultilineLogStringNative : public Processor { std::string mSourceKey = DEFAULT_CONTENT_KEY; MultilineOptions mMultiline; - bool mAppendingLogPositionMeta = false; bool mEnableRawContent = false; const std::string& Name() const override { return sName; } diff --git a/core/plugin/processor/inner/ProcessorTagNative.cpp b/core/plugin/processor/inner/ProcessorTagNative.cpp index 7c2d27a5d4..1bbce38ac4 100644 --- a/core/plugin/processor/inner/ProcessorTagNative.cpp +++ b/core/plugin/processor/inner/ProcessorTagNative.cpp @@ -18,9 +18,11 @@ #include +#include "TagConstants.h" #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/Flags.h" +#include "models/PipelineEventGroup.h" #include "monitor/Monitor.h" #include "pipeline/Pipeline.h" #include "protobuf/sls/sls_logs.pb.h" @@ -47,11 +49,7 @@ bool ProcessorTagNative::Init(const Json::Value& config) { void ProcessorTagNative::Process(PipelineEventGroup& logGroup) { #ifdef __ENTERPRISE__ - string agentTag = EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(); - if (!agentTag.empty()) { - auto sb = logGroup.GetSourceBuffer()->CopyString(agentTag); - addTagIfRequired(logGroup, "AGENT_TAG", AGENT_TAG_DEFAULT_KEY, StringView(sb.data, sb.size)); - } + addDefaultAddedTag(logGroup, TagKey::AGENT_TAG, EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet()); #endif if (!STRING_FLAG(ALIYUN_LOG_FILE_TAGS).empty()) { @@ -66,10 +64,9 @@ void ProcessorTagNative::Process(PipelineEventGroup& logGroup) { if (mContext->GetPipeline().IsFlushingThroughGoPipeline()) { return; } + addDefaultAddedTag(logGroup, TagKey::HOST_NAME, LoongCollectorMonitor::GetInstance()->mHostname); + addDefaultAddedTag(logGroup, TagKey::HOST_IP, LoongCollectorMonitor::GetInstance()->mIpAddr); - // process level - logGroup.SetTagNoCopy(LOG_RESERVED_KEY_HOSTNAME, LoongCollectorMonitor::mHostname); - logGroup.SetTagNoCopy(LOG_RESERVED_KEY_SOURCE, LoongCollectorMonitor::mIpAddr); auto sb = logGroup.GetSourceBuffer()->CopyString(Application::GetInstance()->GetUUID()); logGroup.SetTagNoCopy(LOG_RESERVED_KEY_MACHINE_UUID, StringView(sb.data, sb.size)); static const vector& sEnvTags = AppConfig::GetInstance()->GetEnvTags(); @@ -95,22 +92,35 @@ bool ProcessorTagNative::IsSupportedEvent(const PipelineEventPtr& /*e*/) const { return true; } -void ProcessorTagNative::addTagIfRequired(PipelineEventGroup& logGroup, - const std::string& configKey, - const std::string& defaultKey, - const StringView& value) const { - auto it = mPipelineMetaTagKey.find(configKey); +void ProcessorTagNative::addDefaultAddedTag(PipelineEventGroup& logGroup, TagKey tagKey, const string& value) const { + auto sb = logGroup.GetSourceBuffer()->CopyString(value); + auto it = mPipelineMetaTagKey.find(tagKey); if (it != mPipelineMetaTagKey.end()) { if (!it->second.empty()) { if (it->second == DEFAULT_CONFIG_TAG_KEY_VALUE) { - logGroup.SetTagNoCopy(defaultKey, value); + logGroup.SetTagNoCopy(TagKeyToString(tagKey), StringView(sb.data, sb.size)); } else { - logGroup.SetTagNoCopy(it->second, value); + logGroup.SetTagNoCopy(it->second, StringView(sb.data, sb.size)); } } - // emtpy value means not set + // emtpy value means delete } else { - logGroup.SetTagNoCopy(defaultKey, value); + logGroup.SetTagNoCopy(TagKeyToString(tagKey), StringView(sb.data, sb.size)); + } +} + +void ProcessorTagNative::addOptionalTag(PipelineEventGroup& logGroup, TagKey tagKey, const string& value) const { + auto sb = logGroup.GetSourceBuffer()->CopyString(value); + auto it = mPipelineMetaTagKey.find(tagKey); + if (it != mPipelineMetaTagKey.end()) { + if (!it->second.empty()) { + if (it->second == DEFAULT_CONFIG_TAG_KEY_VALUE) { + logGroup.SetTagNoCopy(TagKeyToString(tagKey), StringView(sb.data, sb.size)); + } else { + logGroup.SetTagNoCopy(it->second, StringView(sb.data, sb.size)); + } + } + // emtpy value means delete } } diff --git a/core/plugin/processor/inner/ProcessorTagNative.h b/core/plugin/processor/inner/ProcessorTagNative.h index 74b506dd07..5f2f4c73e1 100644 --- a/core/plugin/processor/inner/ProcessorTagNative.h +++ b/core/plugin/processor/inner/ProcessorTagNative.h @@ -16,6 +16,7 @@ #pragma once +#include "TagConstants.h" #include "pipeline/plugin/interface/Processor.h" namespace logtail { @@ -32,11 +33,9 @@ class ProcessorTagNative : public Processor { bool IsSupportedEvent(const PipelineEventPtr& e) const override; private: - void addTagIfRequired(PipelineEventGroup& logGroup, - const std::string& configKey, - const std::string& defaultKey, - const StringView& value) const; - std::unordered_map mPipelineMetaTagKey; + void addDefaultAddedTag(PipelineEventGroup& logGroup, TagKey tagKey, const std::string& value) const; + void addOptionalTag(PipelineEventGroup& logGroup, TagKey tagKey, const std::string& value) const; + std::unordered_map mPipelineMetaTagKey; #ifdef __ENTERPRISE__ bool mEnableAgentEnvMetaTagControl = false; std::unordered_map mAgentEnvMetaTagKey; diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 8f4f3d4d04..d9e937fe57 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -210,13 +210,14 @@ bool ProcessorRunner::Serialize( return false; } } - if (group.HasMetadata(EventGroupMetaKey::TOPIC)) { - logGroup.set_topic(group.GetMetadata(EventGroupMetaKey::TOPIC).to_string()); - } for (const auto& tag : group.GetTags()) { - auto logTag = logGroup.add_logtags(); - logTag->set_key(tag.first.to_string()); - logTag->set_value(tag.second.to_string()); + if (tag.first == LOG_RESERVED_KEY_TOPIC) { + logGroup.set_topic(tag.second.to_string()); + } else { + auto logTag = logGroup.add_logtags(); + logTag->set_key(tag.first.to_string()); + logTag->set_value(tag.second.to_string()); + } } logGroup.set_category(logstore); size_t size = logGroup.ByteSizeLong(); diff --git a/core/unittest/batch/BatchItemUnittest.cpp b/core/unittest/batch/BatchItemUnittest.cpp index db44d471d8..b60c0e356f 100644 --- a/core/unittest/batch/BatchItemUnittest.cpp +++ b/core/unittest/batch/BatchItemUnittest.cpp @@ -41,8 +41,7 @@ class EventBatchItemUnittest : public ::testing::Test { void SetUp() override { StringBuffer b = sEventGroup->GetSourceBuffer()->CopyString(string("pack_id")); - mItem.Reset(sEventGroup->GetAllMetadata(), - sEventGroup->GetSizedTags(), + mItem.Reset(sEventGroup->GetSizedTags(), sEventGroup->GetSourceBuffer(), sEventGroup->GetExactlyOnceCheckpoint(), StringView(b.data, b.size)); diff --git a/core/unittest/file_source/FileTagOptionsUnittest.cpp b/core/unittest/file_source/FileTagOptionsUnittest.cpp old mode 100644 new mode 100755 index 77ac1c40b7..cf0041ed60 --- a/core/unittest/file_source/FileTagOptionsUnittest.cpp +++ b/core/unittest/file_source/FileTagOptionsUnittest.cpp @@ -49,7 +49,7 @@ void FileTagOptionsUnittest::OnSuccessfulInit() const { config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, false)); APSARA_TEST_EQUAL(1, config->mFileTags.size()); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); // AppendingLogPositionMeta configStr = R"( @@ -61,9 +61,9 @@ void FileTagOptionsUnittest::OnSuccessfulInit() const { config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, false)); APSARA_TEST_EQUAL(3, config->mFileTags.size()); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_OFFSET_KEY], TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_INODE_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_OFFSET_KEY], TagKeyToString(TagKey::FILE_OFFSET_KEY)); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_INODE_TAG_KEY], TagKeyToString(TagKey::FILE_INODE_TAG_KEY)); configStr = R"( { @@ -78,7 +78,7 @@ void FileTagOptionsUnittest::OnSuccessfulInit() const { config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, false)); APSARA_TEST_EQUAL(3, config->mFileTags.size()); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_OFFSET_KEY], "test_offset"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_INODE_TAG_KEY], "test_inode"); @@ -95,7 +95,7 @@ void FileTagOptionsUnittest::OnSuccessfulInit() const { config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, false)); APSARA_TEST_EQUAL(3, config->mFileTags.size()); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_OFFSET_KEY], "test_offset"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_INODE_TAG_KEY], "test_inode"); @@ -108,25 +108,16 @@ void FileTagOptionsUnittest::OnSuccessfulInit() const { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, true)); - APSARA_TEST_EQUAL(10, config->mFileTags.size()); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_NAMESPACE_TAG_KEY], - TagKeyDefaultValue[TagKey::K8S_NAMESPACE_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_POD_NAME_TAG_KEY], - TagKeyDefaultValue[TagKey::K8S_POD_NAME_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_POD_UID_TAG_KEY], TagKeyDefaultValue[TagKey::K8S_POD_UID_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_CONTAINER_NAME_TAG_KEY], - TagKeyDefaultValue[TagKey::K8S_CONTAINER_NAME_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_CONTAINER_IP_TAG_KEY], - TagKeyDefaultValue[TagKey::K8S_CONTAINER_IP_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_CONTAINER_IMAGE_NAME_TAG_KEY], - TagKeyDefaultValue[TagKey::K8S_CONTAINER_IMAGE_NAME_TAG_KEY]); + APSARA_TEST_EQUAL(7, config->mFileTags.size()); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_NAMESPACE_TAG_KEY], TagKeyToString(TagKey::K8S_NAMESPACE_TAG_KEY)); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_POD_NAME_TAG_KEY], TagKeyToString(TagKey::K8S_POD_NAME_TAG_KEY)); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_POD_UID_TAG_KEY], TagKeyToString(TagKey::K8S_POD_UID_TAG_KEY)); APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_NAME_TAG_KEY], - TagKeyDefaultValue[TagKey::CONTAINER_NAME_TAG_KEY]); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_IP_TAG_KEY], - TagKeyDefaultValue[TagKey::CONTAINER_IP_TAG_KEY]); + TagKeyToString(TagKey::CONTAINER_NAME_TAG_KEY)); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_IP_TAG_KEY], TagKeyToString(TagKey::CONTAINER_IP_TAG_KEY)); APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_IMAGE_NAME_TAG_KEY], - TagKeyDefaultValue[TagKey::CONTAINER_IMAGE_NAME_TAG_KEY]); + TagKeyToString(TagKey::CONTAINER_IMAGE_NAME_TAG_KEY)); configStr = R"( { @@ -144,14 +135,11 @@ void FileTagOptionsUnittest::OnSuccessfulInit() const { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, true)); - APSARA_TEST_EQUAL(10, config->mFileTags.size()); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]); + APSARA_TEST_EQUAL(7, config->mFileTags.size()); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_NAMESPACE_TAG_KEY], "test_namespace"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_POD_NAME_TAG_KEY], "test_pod_name"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_POD_UID_TAG_KEY], "test_pod_uid"); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_CONTAINER_NAME_TAG_KEY], "test_container_name"); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_CONTAINER_IP_TAG_KEY], "test_container_ip"); - APSARA_TEST_EQUAL(config->mFileTags[TagKey::K8S_CONTAINER_IMAGE_NAME_TAG_KEY], "test_container_image_name"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_NAME_TAG_KEY], "test_container_name"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_IP_TAG_KEY], "test_container_ip"); APSARA_TEST_EQUAL(config->mFileTags[TagKey::CONTAINER_IMAGE_NAME_TAG_KEY], "test_container_image_name"); @@ -202,7 +190,8 @@ void FileTagOptionsUnittest::OnInvalidInit() const { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); config.reset(new FileTagOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, false)); - APSARA_TEST_EQUAL(0, config->mFileTags.size()); + APSARA_TEST_EQUAL(1, config->mFileTags.size()); + APSARA_TEST_EQUAL(config->mFileTags[TagKey::FILE_PATH_TAG_KEY], TagKeyToString(TagKey::FILE_PATH_TAG_KEY)); } UNIT_TEST_CASE(FileTagOptionsUnittest, OnSuccessfulInit) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index fe23eff0c9..4e6b7d35ed 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -17,6 +17,7 @@ #include "json/json.h" +#include "TagConstants.h" #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" @@ -52,8 +53,6 @@ using namespace std; namespace logtail { -const string TOPIC_VALUE = "topic"; - class FlusherSLSUnittest : public testing::Test { public: void OnSuccessfulInit(); @@ -1203,11 +1202,9 @@ void FlusherSLSUnittest::TestSend() { // replayed group PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("172.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetTag(TagKeyDefaultValue[TagKey::HOST_NAME], "hostname"); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); auto cpt = make_shared(); cpt->index = 1; cpt->fbKey = eooKey; @@ -1244,10 +1241,8 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("topic", logGroup.topic()); APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); - APSARA_TEST_EQUAL(2, logGroup.logtags_size()); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(0).key()); - APSARA_TEST_EQUAL(TagKeyDefaultValue[TagKey::HOST_NAME], logGroup.logtags(1).key()); - APSARA_TEST_EQUAL("hostname", logGroup.logtags(1).value()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); @@ -1261,11 +1256,9 @@ void FlusherSLSUnittest::TestSend() { flusher.mBatcher.GetEventFlushStrategy().SetMinCnt(1); PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("172.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetTag(TagKeyDefaultValue[TagKey::HOST_NAME], "hostname"); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); auto cpt = make_shared(); cpt->fbKey = eooKey; cpt->data.set_read_offset(0); @@ -1299,10 +1292,8 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("topic", logGroup.topic()); APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); - APSARA_TEST_EQUAL(2, logGroup.logtags_size()); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(0).key()); - APSARA_TEST_EQUAL(TagKeyDefaultValue[TagKey::HOST_NAME], logGroup.logtags(1).key()); - APSARA_TEST_EQUAL("hostname", logGroup.logtags(1).value()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); @@ -1347,10 +1338,9 @@ void FlusherSLSUnittest::TestSend() { flusher.mBatcher.GetEventFlushStrategy().SetMinCnt(1); PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("172.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetTag(TagKeyDefaultValue[TagKey::HOST_NAME], "hostname"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); group.SetTag(string("tag_key"), string("tag_value")); auto e = group.AddLogEvent(); e->SetTimestamp(1234567890); @@ -1379,12 +1369,10 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("topic", logGroup.topic()); APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); - APSARA_TEST_EQUAL(3, logGroup.logtags_size()); + APSARA_TEST_EQUAL(2, logGroup.logtags_size()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(0).key()); - APSARA_TEST_EQUAL(TagKeyDefaultValue[TagKey::HOST_NAME], logGroup.logtags(1).key()); - APSARA_TEST_EQUAL("hostname", logGroup.logtags(1).value()); - APSARA_TEST_EQUAL("tag_key", logGroup.logtags(2).key()); - APSARA_TEST_EQUAL("tag_value", logGroup.logtags(2).value()); + APSARA_TEST_EQUAL("tag_key", logGroup.logtags(1).key()); + APSARA_TEST_EQUAL("tag_value", logGroup.logtags(1).value()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); @@ -1429,10 +1417,9 @@ void FlusherSLSUnittest::TestSend() { PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("172.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetTag(TagKeyDefaultValue[TagKey::HOST_NAME], "hostname"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); { auto e = group.AddLogEvent(); e->SetTimestamp(1234567890); @@ -1483,10 +1470,8 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("topic", logGroup.topic()); APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); - APSARA_TEST_EQUAL(2, logGroup.logtags_size()); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(0).key()); - APSARA_TEST_EQUAL(TagKeyDefaultValue[TagKey::HOST_NAME], logGroup.logtags(1).key()); - APSARA_TEST_EQUAL("hostname", logGroup.logtags(1).value()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); if (i == 0) { APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -1531,10 +1516,9 @@ void FlusherSLSUnittest::TestFlush() { PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("172.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetTag(TagKeyDefaultValue[TagKey::HOST_NAME], "hostname"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); { auto e = group.AddLogEvent(); e->SetTimestamp(1234567890); @@ -1575,10 +1559,9 @@ void FlusherSLSUnittest::TestFlushAll() { PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("172.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetTag(TagKeyDefaultValue[TagKey::HOST_NAME], "hostname"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); { auto e = group.AddLogEvent(); e->SetTimestamp(1234567890); diff --git a/core/unittest/input/InputFileUnittest.cpp b/core/unittest/input/InputFileUnittest.cpp index 108ea63c59..7acbefa985 100644 --- a/core/unittest/input/InputFileUnittest.cpp +++ b/core/unittest/input/InputFileUnittest.cpp @@ -255,7 +255,6 @@ void InputFileUnittest::TestCreateInnerProcessors() { auto plugin = static_cast(input->mInnerProcessors[0]->mPlugin.get()); APSARA_TEST_EQUAL(DEFAULT_CONTENT_KEY, plugin->mSourceKey); APSARA_TEST_EQUAL('\n', plugin->mSplitChar); - APSARA_TEST_TRUE(plugin->mAppendingLogPositionMeta); APSARA_TEST_FALSE(plugin->mEnableRawContent); APSARA_TEST_EQUAL(ProcessorTagNative::sName, input->mInnerProcessors[1]->Name()); } @@ -291,7 +290,6 @@ void InputFileUnittest::TestCreateInnerProcessors() { APSARA_TEST_TRUE(plugin->mMultiline.mIgnoringUnmatchWarning); APSARA_TEST_EQUAL(MultilineOptions::UnmatchedContentTreatment::DISCARD, plugin->mMultiline.mUnmatchedContentTreatment); - APSARA_TEST_TRUE(plugin->mAppendingLogPositionMeta); APSARA_TEST_FALSE(plugin->mEnableRawContent); APSARA_TEST_EQUAL(ProcessorTagNative::sName, input->mInnerProcessors[1]->Name()); } @@ -316,7 +314,6 @@ void InputFileUnittest::TestCreateInnerProcessors() { auto plugin = static_cast(input->mInnerProcessors[0]->mPlugin.get()); APSARA_TEST_EQUAL(DEFAULT_CONTENT_KEY, plugin->mSourceKey); APSARA_TEST_EQUAL('\0', plugin->mSplitChar); - APSARA_TEST_TRUE(plugin->mAppendingLogPositionMeta); APSARA_TEST_FALSE(plugin->mEnableRawContent); APSARA_TEST_EQUAL(ProcessorTagNative::sName, input->mInnerProcessors[1]->Name()); ctx.SetIsFirstProcessorJsonFlag(false); @@ -345,7 +342,6 @@ void InputFileUnittest::TestCreateInnerProcessors() { auto plugin = static_cast(input->mInnerProcessors[0]->mPlugin.get()); APSARA_TEST_EQUAL(DEFAULT_CONTENT_KEY, plugin->mSourceKey); APSARA_TEST_EQUAL('\0', plugin->mSplitChar); - APSARA_TEST_TRUE(plugin->mAppendingLogPositionMeta); APSARA_TEST_FALSE(plugin->mEnableRawContent); APSARA_TEST_EQUAL(ProcessorTagNative::sName, input->mInnerProcessors[1]->Name()); ctx.SetIsFirstProcessorJsonFlag(false); diff --git a/core/unittest/metadata/K8sMetadataUnittest.cpp b/core/unittest/metadata/K8sMetadataUnittest.cpp index 26e76a7914..9b53ef344a 100644 --- a/core/unittest/metadata/K8sMetadataUnittest.cpp +++ b/core/unittest/metadata/K8sMetadataUnittest.cpp @@ -164,7 +164,7 @@ class k8sMetadataUnittest : public ::testing::Test { ], "metadata" : { - "log.file.path" : "/var/log/message" + "log.file.path_resolved" : "/var/log/message" }, "tags" : { @@ -399,7 +399,7 @@ class k8sMetadataUnittest : public ::testing::Test { ], "metadata" : { - "log.file.path" : "/var/log/message" + "log.file.path_resolved" : "/var/log/message" }, "tags" : { diff --git a/core/unittest/models/PipelineEventGroupUnittest.cpp b/core/unittest/models/PipelineEventGroupUnittest.cpp index 3a308faca5..5b71a39d05 100644 --- a/core/unittest/models/PipelineEventGroupUnittest.cpp +++ b/core/unittest/models/PipelineEventGroupUnittest.cpp @@ -217,7 +217,7 @@ void PipelineEventGroupUnittest::TestCopy() { void PipelineEventGroupUnittest::TestSetMetadata() { { // string copy, let kv out of scope - mEventGroup->SetMetadata(EventGroupMetaKey::LOG_FILE_PATH, std::string("value1")); + mEventGroup->SetMetadata(EventGroupMetaKey::LOG_FORMAT, std::string("value1")); } { // stringview copy, let kv out of scope std::string value("value2"); @@ -227,19 +227,19 @@ void PipelineEventGroupUnittest::TestSetMetadata() { { // StringBuffer nocopy StringBuffer value = mEventGroup->GetSourceBuffer()->CopyString(std::string("value3")); beforeAlloc = mSourceBuffer->mAllocator.TotalAllocated(); - mEventGroup->SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, value); + mEventGroup->SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_OFFSET_KEY, value); } std::string value("value4"); { // StringView nocopy - mEventGroup->SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(value)); + mEventGroup->SetMetadataNoCopy(EventGroupMetaKey::HAS_PART_LOG, StringView(value)); } size_t afterAlloc = mSourceBuffer->mAllocator.TotalAllocated(); APSARA_TEST_EQUAL_FATAL(beforeAlloc, afterAlloc); std::vector> answers - = {{EventGroupMetaKey::LOG_FILE_PATH, "value1"}, + = {{EventGroupMetaKey::LOG_FORMAT, "value1"}, {EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, "value2"}, - {EventGroupMetaKey::LOG_FILE_INODE, "value3"}, - {EventGroupMetaKey::SOURCE_ID, "value4"}}; + {EventGroupMetaKey::LOG_FILE_OFFSET_KEY, "value3"}, + {EventGroupMetaKey::HAS_PART_LOG, "value4"}}; for (const auto kv : answers) { APSARA_TEST_TRUE_FATAL(mEventGroup->HasMetadata(kv.first)); APSARA_TEST_STREQ_FATAL(kv.second.c_str(), mEventGroup->GetMetadata(kv.first).data()); @@ -247,10 +247,10 @@ void PipelineEventGroupUnittest::TestSetMetadata() { } void PipelineEventGroupUnittest::TestDelMetadata() { - mEventGroup->SetMetadata(EventGroupMetaKey::LOG_FILE_INODE, std::string("value1")); - APSARA_TEST_TRUE_FATAL(mEventGroup->HasMetadata(EventGroupMetaKey::LOG_FILE_INODE)); - mEventGroup->DelMetadata(EventGroupMetaKey::LOG_FILE_INODE); - APSARA_TEST_FALSE_FATAL(mEventGroup->HasMetadata(EventGroupMetaKey::LOG_FILE_INODE)); + mEventGroup->SetMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, std::string("value1")); + APSARA_TEST_TRUE_FATAL(mEventGroup->HasMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED)); + mEventGroup->DelMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED); + APSARA_TEST_FALSE_FATAL(mEventGroup->HasMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED)); } void PipelineEventGroupUnittest::TestFromJsonToJson() { @@ -270,7 +270,7 @@ void PipelineEventGroupUnittest::TestFromJsonToJson() { ], "metadata" : { - "log.file.path" : "/var/log/message" + "log.file.path_resolved" : "/var/log/message" }, "tags" : { @@ -283,8 +283,9 @@ void PipelineEventGroupUnittest::TestFromJsonToJson() { auto& logEvent = events[0]; APSARA_TEST_TRUE_FATAL(logEvent.Is()); - APSARA_TEST_TRUE_FATAL(mEventGroup->HasMetadata(EventGroupMetaKey::LOG_FILE_PATH)); - APSARA_TEST_STREQ_FATAL("/var/log/message", mEventGroup->GetMetadata(EventGroupMetaKey::LOG_FILE_PATH).data()); + APSARA_TEST_TRUE_FATAL(mEventGroup->HasMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED)); + APSARA_TEST_STREQ_FATAL("/var/log/message", + mEventGroup->GetMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED).data()); APSARA_TEST_TRUE_FATAL(mEventGroup->HasTag("app_name")); APSARA_TEST_STREQ_FATAL("xxx", mEventGroup->GetTag("app_name").data()); diff --git a/core/unittest/pipeline/GlobalConfigUnittest.cpp b/core/unittest/pipeline/GlobalConfigUnittest.cpp index 105ebf45b0..da142a58cb 100644 --- a/core/unittest/pipeline/GlobalConfigUnittest.cpp +++ b/core/unittest/pipeline/GlobalConfigUnittest.cpp @@ -19,6 +19,7 @@ #include "json/json.h" +#include "TagConstants.h" #include "common/JsonUtil.h" #include "pipeline/GlobalConfig.h" #include "unittest/Unittest.h" @@ -66,8 +67,8 @@ void GlobalConfigUnittest::OnSuccessfulInit() const { "EnableTimestampNanosecond": true, "UsingOldContentTag": true, "PipelineMetaTagKey": { - "key1": "value1", - "key2": "value2" + "HOST_NAME": "value1", + "HOST_ID": "value2" }, "AgentEnvMetaTagKey": { "key3": "value3", @@ -85,8 +86,8 @@ void GlobalConfigUnittest::OnSuccessfulInit() const { APSARA_TEST_TRUE(config->mEnableTimestampNanosecond); APSARA_TEST_TRUE(config->mUsingOldContentTag); APSARA_TEST_EQUAL(config->mPipelineMetaTagKey.size(), 2); - APSARA_TEST_EQUAL(config->mPipelineMetaTagKey["key1"], "value1"); - APSARA_TEST_EQUAL(config->mPipelineMetaTagKey["key2"], "value2"); + APSARA_TEST_EQUAL(config->mPipelineMetaTagKey[TagKey::HOST_NAME], "value1"); + APSARA_TEST_EQUAL(config->mPipelineMetaTagKey[TagKey::HOST_ID], "value2"); #ifdef __ENTERPRISE__ APSARA_TEST_EQUAL(config->mAgentEnvMetaTagKey.size(), 2); APSARA_TEST_EQUAL(config->mAgentEnvMetaTagKey["key3"], "value3"); diff --git a/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp b/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp index ef9f7edb56..43cbf61554 100644 --- a/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp @@ -17,7 +17,6 @@ #include #include "common/JsonUtil.h" -#include "common/TagConstants.h" #include "config/PipelineConfig.h" #include "constants/Constants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" @@ -64,7 +63,7 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessJson() { // make events auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY, TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY, TagKeyToString(TagKey::FILE_OFFSET_KEY)); std::stringstream inJson; inJson << R"({ "events" : @@ -100,11 +99,13 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessJson() { "contents" : { "content" : "{\n\"k1\":\"v1\"\n}", - "log.file.offset": "1" + ")" + + DEFAULT_LOG_TAG_FILE_OFFSET + R"(": "1" }, "fileOffset": 1, "rawSize": )" - << strlen(R"({n"k1":"v1"n}0)") << R"(, + << strlen(R"({n"k1":"v1"n}0)") + << R"(, "timestamp" : 12345678901, "timestampNanosecond" : 0, "type" : 1 @@ -113,20 +114,23 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessJson() { "contents" : { "content" : "{\n\"k2\":\"v2\"\n}", - "log.file.offset": ")" + ")" + + DEFAULT_LOG_TAG_FILE_OFFSET + R"(": ")" << strlen(R"({n"k1":"v1"n}0)") + 1 << R"(" }, "fileOffset": )" << strlen(R"({n"k1":"v1"n}0)") + 1 << R"(, "rawSize": )" - << strlen(R"({n"k2":"v2"n})") << R"(, + << strlen(R"({n"k2":"v2"n})") + << R"(, "timestamp" : 12345678901, "timestampNanosecond" : 0, "type" : 1 } ], "metadata": { - "log.file.offset": "log.file.offset" + "log.file.offset": ")" + + DEFAULT_LOG_TAG_FILE_OFFSET + R"(" } })"; std::string outJson = logGroupList[0].ToJsonString(true); diff --git a/core/unittest/processor/ProcessorTagNativeUnittest.cpp b/core/unittest/processor/ProcessorTagNativeUnittest.cpp index d8035267fc..bb42c2ffd7 100644 --- a/core/unittest/processor/ProcessorTagNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorTagNativeUnittest.cpp @@ -14,6 +14,8 @@ #include +#include "TagConstants.h" +#include "common/JsonUtil.h" #include "config/PipelineConfig.h" #include "constants/Constants.h" #include "file_server/ConfigManager.h" @@ -34,7 +36,6 @@ class ProcessorTagNativeUnittest : public ::testing::Test { protected: void SetUp() override { - mContext.SetConfigName("project##config_0"); LoongCollectorMonitor::GetInstance(); #ifdef __ENTERPRISE__ EnterpriseConfigProvider::GetInstance()->SetUserDefinedIdSet(std::vector{"machine_group"}); @@ -82,12 +83,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); - std::string resolvedFilePath = "/run/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); + eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -135,12 +132,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -188,12 +181,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -241,12 +230,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -260,11 +245,6 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE_FATAL(processor.Init(processorConfig)); processor.Process(eventGroup); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_PATH)); - APSARA_TEST_EQUAL_FATAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH), - eventGroup.GetTag(LOG_RESERVED_KEY_PATH)); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_HOSTNAME)); - APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::mHostname, eventGroup.GetTag(LOG_RESERVED_KEY_HOSTNAME)); #ifdef __ENTERPRISE__ APSARA_TEST_FALSE_FATAL(eventGroup.HasTag("test_agent_tag")); #else @@ -291,12 +271,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -310,15 +286,17 @@ void ProcessorTagNativeUnittest::TestProcess() { processor.Process(eventGroup); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyDefaultValue[TagKey::HOST_NAME])); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mHostname, eventGroup.GetTag(TagKeyDefaultValue[TagKey::HOST_NAME])); + APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::HOST_NAME))); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::GetInstance()->mHostname, + eventGroup.GetTag(TagKeyToString(TagKey::HOST_NAME))); #ifdef __ENTERPRISE__ - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(AGENT_TAG_DEFAULT_KEY)); + APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::AGENT_TAG))); APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), - eventGroup.GetTag(AGENT_TAG_DEFAULT_KEY)); + eventGroup.GetTag(TagKeyToString(TagKey::AGENT_TAG))); #else - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(HOST_IP_DEFAULT_KEY)); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mIpAddr, eventGroup.GetTag(HOST_IP_DEFAULT_KEY)); + APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::HOST_IP))); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::GetInstance()->mIpAddr, + eventGroup.GetTag(TagKeyToString(TagKey::HOST_IP))); #endif } { // native branch default @@ -348,12 +326,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -366,15 +340,17 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE_FATAL(processor.Init(processorConfig)); processor.Process(eventGroup); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyDefaultValue[TagKey::HOST_NAME])); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mHostname, eventGroup.GetTag(TagKeyDefaultValue[TagKey::HOST_NAME])); + APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::HOST_NAME))); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::GetInstance()->mHostname, + eventGroup.GetTag(TagKeyToString(TagKey::HOST_NAME))); #ifdef __ENTERPRISE__ - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(AGENT_TAG_DEFAULT_KEY)); + APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::AGENT_TAG))); APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), - eventGroup.GetTag(AGENT_TAG_DEFAULT_KEY)); + eventGroup.GetTag(TagKeyToString(TagKey::AGENT_TAG))); #else - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(HOST_IP_DEFAULT_KEY)); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mIpAddr, eventGroup.GetTag(HOST_IP_DEFAULT_KEY)); + APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::HOST_IP))); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::GetInstance()->mIpAddr, + eventGroup.GetTag(TagKeyToString(TagKey::HOST_IP))); #endif } { // native branch rename @@ -404,12 +380,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -423,14 +395,14 @@ void ProcessorTagNativeUnittest::TestProcess() { processor.Process(eventGroup); APSARA_TEST_TRUE_FATAL(eventGroup.HasTag("test_host_name")); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mHostname, eventGroup.GetTag("test_host_name")); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::GetInstance()->mHostname, eventGroup.GetTag("test_host_name")); #ifdef __ENTERPRISE__ APSARA_TEST_TRUE_FATAL(eventGroup.HasTag("test_agent_tag")); APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), eventGroup.GetTag("test_agent_tag")); #else APSARA_TEST_TRUE_FATAL(eventGroup.HasTag("test_host_ip")); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mIpAddr, eventGroup.GetTag("test_host_ip")); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::GetInstance()->mIpAddr, eventGroup.GetTag("test_host_ip")); #endif } { // native branch delete @@ -460,12 +432,8 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); - std::string filePath = "/var/log/message"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH, filePath); std::string resolvedFilePath = "/run/var/log/message"; eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED, resolvedFilePath); - std::string inode = "123456"; - eventGroup.SetMetadataNoCopy(EventGroupMetaKey::LOG_FILE_INODE, inode); Pipeline pipeline; PipelineContext context; context.SetConfigName("project##config_0"); @@ -478,11 +446,11 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE_FATAL(processor.Init(processorConfig)); processor.Process(eventGroup); - APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(TagKeyDefaultValue[TagKey::HOST_NAME])); + APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::HOST_NAME))); #ifdef __ENTERPRISE__ - APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(AGENT_TAG_DEFAULT_KEY)); + APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::AGENT_TAG))); #else - APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(HOST_IP_DEFAULT_KEY)); + APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(TagKeyToString(TagKey::HOST_IP))); #endif } } diff --git a/core/unittest/reader/DeletedFileUnittest.cpp b/core/unittest/reader/DeletedFileUnittest.cpp index c9e3180e67..da5dc6268a 100644 --- a/core/unittest/reader/DeletedFileUnittest.cpp +++ b/core/unittest/reader/DeletedFileUnittest.cpp @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "file_server/reader/LogFileReader.h" #include "unittest/Unittest.h" @@ -32,7 +34,8 @@ class DeletedFileUnittest : public testing::Test { hostLogPathFile, DevInode(), make_pair(&readerOpts, &ctx), - make_pair(&multilineOpts, &ctx))); + make_pair(&multilineOpts, &ctx), + make_pair(nullptr, &ctx))); } void TearDown() override { INT32_FLAG(force_release_deleted_file_fd_timeout) = -1; } diff --git a/core/unittest/reader/FileTagUnittest.cpp b/core/unittest/reader/FileTagUnittest.cpp old mode 100644 new mode 100755 index 85a08ac1f7..93815df167 --- a/core/unittest/reader/FileTagUnittest.cpp +++ b/core/unittest/reader/FileTagUnittest.cpp @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#include "Constants.h" +#include "TagConstants.h" #include "common/JsonUtil.h" #include "file_server/reader/LogFileReader.h" #include "unittest/Unittest.h" @@ -32,60 +36,18 @@ class FileTagUnittest : public testing::Test { void TearDown() override {} private: - vector GenerateFakeContainerTags() { - vector tags; - auto logTag_1 = sls_logs::LogTag(); - logTag_1.set_key("_image_name_"); - logTag_1.set_value("test_image"); - tags.push_back(logTag_1); - auto logTag_2 = sls_logs::LogTag(); - logTag_2.set_key("_container_name_"); - logTag_2.set_value("test_container"); - tags.push_back(logTag_2); - auto logTag_3 = sls_logs::LogTag(); - logTag_3.set_key("_container_ip_"); - logTag_3.set_value("test_container_ip"); - tags.push_back(logTag_3); - // env tags, like __aliyun_logs_xxx_tags__ - auto logTag_4 = sls_logs::LogTag(); - logTag_4.set_key("_test_tag_"); - logTag_4.set_value("test_value"); - tags.push_back(logTag_4); - return tags; + vector> GenerateFakeContainerMetadatas() { + vector> metadata; + metadata.emplace_back(TagKey::CONTAINER_IMAGE_NAME_TAG_KEY, "test_image"); + metadata.emplace_back(TagKey::CONTAINER_NAME_TAG_KEY, "test_container"); + metadata.emplace_back(TagKey::CONTAINER_IP_TAG_KEY, "test_container_ip"); + return metadata; } - vector GenerateFakeK8sContainerTags() { - vector tags; - auto logTag_1 = sls_logs::LogTag(); - logTag_1.set_key("_k8s_image_name_"); - logTag_1.set_value("test_k8s_image"); - tags.push_back(logTag_1); - auto logTag_2 = sls_logs::LogTag(); - logTag_2.set_key("_k8s_container_name_"); - logTag_2.set_value("test_k8s_container"); - tags.push_back(logTag_2); - auto logTag_3 = sls_logs::LogTag(); - logTag_3.set_key("_pod_name_"); - logTag_3.set_value("test_pod"); - tags.push_back(logTag_3); - auto logTag_4 = sls_logs::LogTag(); - logTag_4.set_key("_namespace_"); - logTag_4.set_value("test_namespace"); - tags.push_back(logTag_4); - auto logTag_5 = sls_logs::LogTag(); - logTag_5.set_key("_pod_uid_"); - logTag_5.set_value("test_pod_uid"); - tags.push_back(logTag_5); - auto logTag_6 = sls_logs::LogTag(); - logTag_6.set_key("_k8s_container_ip_"); - logTag_6.set_value("test_k8s_container_ip"); - tags.push_back(logTag_6); - // env tags, like __aliyun_logs_xxx_tags__ - auto logTag_7 = sls_logs::LogTag(); - logTag_7.set_key("_test_tag_"); - logTag_7.set_value("test_value"); - tags.push_back(logTag_7); - return tags; + vector> GenerateFakeContainerExtraTags() { + vector> extraTags; + extraTags.emplace_back("_test_tag_", "test_value"); + return extraTags; } FileReaderOptions readerOpts; @@ -117,8 +79,11 @@ void FileTagUnittest::TestDefaultTag() { PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 1); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]), + APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 2); + for (const auto& item : eventGroup.GetTags()) { + std::cout << item.first << " : " << item.second << std::endl; + } + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_PATH_TAG_KEY)), hostLogPathDir + "/" + hostLogPathFile); } { @@ -138,29 +103,24 @@ void FileTagUnittest::TestDefaultTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeContainerTags()); + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; + reader.mContainerMetadatas = GenerateFakeContainerMetadatas(); + reader.mExtraTags = GenerateFakeContainerExtraTags(); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), - TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); + TagKeyToString(TagKey::FILE_OFFSET_KEY)); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 8); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]), + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_PATH_TAG_KEY)), hostLogPathDir + "/" + hostLogPathFile); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]), "0"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::CONTAINER_IMAGE_NAME_TAG_KEY]), "test_image"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::CONTAINER_NAME_TAG_KEY]), "test_container"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::CONTAINER_IP_TAG_KEY]), "test_container_ip"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_INODE_TAG_KEY)), "0"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::CONTAINER_IMAGE_NAME_TAG_KEY)), "test_image"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::CONTAINER_NAME_TAG_KEY)), "test_container"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::CONTAINER_IP_TAG_KEY)), "test_container_ip"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); APSARA_TEST_EQUAL(eventGroup.GetTag("_test_tag_"), "test_value"); @@ -186,25 +146,19 @@ void FileTagUnittest::TestDefaultTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), - TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); + TagKeyToString(TagKey::FILE_OFFSET_KEY)); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 4); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]), + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_PATH_TAG_KEY)), hostLogPathDir + "/" + hostLogPathFile); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]), "0"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_INODE_TAG_KEY)), "0"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); } @@ -229,25 +183,19 @@ void FileTagUnittest::TestDefaultTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), - TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); + TagKeyToString(TagKey::FILE_OFFSET_KEY)); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 4); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]), + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_PATH_TAG_KEY)), hostLogPathDir + "/" + hostLogPathFile); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]), "0"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_INODE_TAG_KEY)), "0"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); } @@ -279,90 +227,27 @@ void FileTagUnittest::TestDefaultTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeK8sContainerTags()); + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; + reader.mContainerMetadatas = GenerateFakeContainerMetadatas(); + reader.mExtraTags = GenerateFakeContainerExtraTags(); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), - TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); + TagKeyToString(TagKey::FILE_OFFSET_KEY)); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 11); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]), - hostLogPathDir + "/" + hostLogPathFile); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]), "0"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::K8S_CONTAINER_IMAGE_NAME_TAG_KEY]), - "test_k8s_image"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::K8S_CONTAINER_NAME_TAG_KEY]), - "test_k8s_container"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::K8S_CONTAINER_IP_TAG_KEY]), - "test_k8s_container_ip"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::K8S_POD_NAME_TAG_KEY]), "test_pod"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::K8S_NAMESPACE_TAG_KEY]), "test_namespace"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::K8S_POD_UID_TAG_KEY]), "test_pod_uid"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); - APSARA_TEST_EQUAL(eventGroup.GetTag("_test_tag_"), "test_value"); - } - { - configStr = R"( - { - "AppendingLogPositionMeta": false, - "EnableContainerDiscovery": true, - "FileOffsetKey": "__default__", - "Tags": { - "FilePathTagKey": "__default__", - "FileInodeTagKey": "__default__", - "K8sNamespaceTagKey": "__default__", - "K8sPodNameTagKey": "__default__", - "K8sPodUidTagKey": "__default__", - "ContainerNameTagKey": "__default__", - "ContainerIpTagKey": "__default__", - "ContainerImageNameTagKey": "__default__" - } - } - )"; - APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - config.reset(new FileTagOptions()); - APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, true)); - LogFileReader reader = LogFileReader(hostLogPathDir, - hostLogPathFile, - DevInode(), - make_pair(&readerOpts, &ctx), - make_pair(&multilineOpts, &ctx), - make_pair(config.get(), &ctx)); - reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeContainerTags()); - - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), - TagKeyDefaultValue[TagKey::FILE_OFFSET_KEY]); - - APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 8); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_PATH_TAG_KEY]), + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_PATH_TAG_KEY)), hostLogPathDir + "/" + hostLogPathFile); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::FILE_INODE_TAG_KEY]), "0"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::CONTAINER_IMAGE_NAME_TAG_KEY]), "test_image"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::CONTAINER_NAME_TAG_KEY]), "test_container"); - APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyDefaultValue[TagKey::CONTAINER_IP_TAG_KEY]), "test_container_ip"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::FILE_INODE_TAG_KEY)), "0"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::CONTAINER_IMAGE_NAME_TAG_KEY)), "test_image"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::CONTAINER_NAME_TAG_KEY)), "test_container"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::CONTAINER_IP_TAG_KEY)), "test_container_ip"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::K8S_POD_NAME_TAG_KEY)), "test_pod"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::K8S_NAMESPACE_TAG_KEY)), "test_namespace"); + APSARA_TEST_EQUAL(eventGroup.GetTag(TagKeyToString(TagKey::K8S_POD_UID_TAG_KEY)), "test_pod_uid"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); APSARA_TEST_EQUAL(eventGroup.GetTag("_test_tag_"), "test_value"); @@ -394,18 +279,12 @@ void FileTagUnittest::TestRenameTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), "test_offset"); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 4); @@ -435,18 +314,12 @@ void FileTagUnittest::TestRenameTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), "test_offset"); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 4); @@ -483,19 +356,14 @@ void FileTagUnittest::TestRenameTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeContainerTags()); + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; + reader.mContainerMetadatas = GenerateFakeContainerMetadatas(); + reader.mExtraTags = GenerateFakeContainerExtraTags(); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), "test_offset"); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 8); @@ -508,62 +376,6 @@ void FileTagUnittest::TestRenameTag() { APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); APSARA_TEST_EQUAL(eventGroup.GetTag("_test_tag_"), "test_value"); } - { - configStr = R"( - { - "AppendingLogPositionMeta": false, - "EnableContainerDiscovery": true, - "FileOffsetKey": "test_offset", - "Tags": { - "FilePathTagKey": "test_path", - "FileInodeTagKey": "test_inode", - "K8sNamespaceTagKey": "test_namespace", - "K8sPodNameTagKey": "test_pod", - "K8sPodUidTagKey": "test_pod_uid", - "ContainerNameTagKey": "test_container", - "ContainerIpTagKey": "test_container_ip", - "ContainerImageNameTagKey": "test_image" - } - } - )"; - APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - config.reset(new FileTagOptions()); - APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, true)); - LogFileReader reader = LogFileReader(hostLogPathDir, - hostLogPathFile, - DevInode(), - make_pair(&readerOpts, &ctx), - make_pair(&multilineOpts, &ctx), - make_pair(config.get(), &ctx)); - reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeK8sContainerTags()); - - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_OFFSET_KEY), "test_offset"); - - APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 11); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_path"), hostLogPathDir + "/" + hostLogPathFile); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_inode"), "0"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_image"), "test_k8s_image"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_container"), "test_k8s_container"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_container_ip"), "test_k8s_container_ip"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_namespace"), "test_namespace"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_pod"), "test_pod"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_pod_uid"), "test_pod_uid"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); - APSARA_TEST_EQUAL(eventGroup.GetTag("_test_tag_"), "test_value"); - } } void FileTagUnittest::TestDeleteTag() { @@ -591,18 +403,12 @@ void FileTagUnittest::TestDeleteTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 2); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); @@ -629,65 +435,17 @@ void FileTagUnittest::TestDeleteTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 2); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); } - { - configStr = R"( - { - "AppendingLogPositionMeta": false, - "EnableContainerDiscovery": true, - "Tags": { - "FilePathTagKey": "", - "ContainerNameTagKey": "", - "ContainerIpTagKey": "", - "ContainerImageNameTagKey": "" - } - } - )"; - APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - config.reset(new FileTagOptions()); - APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType, true)); - LogFileReader reader = LogFileReader(hostLogPathDir, - hostLogPathFile, - DevInode(), - make_pair(&readerOpts, &ctx), - make_pair(&multilineOpts, &ctx), - make_pair(config.get(), &ctx)); - reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeContainerTags()); - - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); - - APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 3); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); - APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_2"), "test_topic_value_2"); - APSARA_TEST_EQUAL(eventGroup.GetTag("_test_tag_"), "test_value"); - } { configStr = R"( { @@ -714,19 +472,14 @@ void FileTagUnittest::TestDeleteTag() { make_pair(&multilineOpts, &ctx), make_pair(config.get(), &ctx)); reader.mTopicName = "test_topic"; - sls_logs::LogTag tag1; - tag1.set_key("test_topic_1"); - tag1.set_value("test_topic_value_1"); - sls_logs::LogTag tag2; - tag2.set_key("test_topic_2"); - tag2.set_value("test_topic_value_2"); - reader.mTopicExtraTags = {tag1, tag2}; - reader.mContainerExtraTags = make_shared>(GenerateFakeK8sContainerTags()); + reader.mTopicExtraTags = {{"test_topic_1", "test_topic_value_1"}, {"test_topic_2", "test_topic_value_2"}}; + reader.mContainerMetadatas = GenerateFakeContainerMetadatas(); + reader.mExtraTags = GenerateFakeContainerExtraTags(); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); reader.SetEventGroupMetaAndTag(eventGroup); - APSARA_TEST_EQUAL(eventGroup.GetMetadata(EventGroupMetaKey::TOPIC), "test_topic"); + APSARA_TEST_EQUAL(eventGroup.GetTag(LOG_RESERVED_KEY_TOPIC), "test_topic"); APSARA_TEST_EQUAL(eventGroup.GetTags().size(), 3); APSARA_TEST_EQUAL(eventGroup.GetTag("test_topic_1"), "test_topic_value_1"); @@ -742,4 +495,4 @@ UNIT_TEST_CASE(FileTagUnittest, TestDeleteTag) } // namespace logtail -UNIT_TEST_MAIN \ No newline at end of file +UNIT_TEST_MAIN diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index f8dd6753d4..5861213074 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -15,6 +15,7 @@ #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" +#include "FileTagOptions.h" #include "common/FileSystemUtil.h" #include "common/memory/SourceBuffer.h" #include "file_server/reader/LogFileReader.h" @@ -520,8 +521,12 @@ UNIT_TEST_CASE(GetLastLineUnittest, TestGetLastLineEmpty); void GetLastLineUnittest::TestGetLastLine() { std::string testLog = "first line\nsecond line\nthird line"; - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(nullptr, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(nullptr, &ctx), + std::make_pair(nullptr, &ctx)); auto lastLine = logFileReader.GetLastLine(const_cast(testLog.data()), testLog.size()); std::string expectLog = "third line"; APSARA_TEST_EQUAL_FATAL(expectLog, std::string(lastLine.data.data(), lastLine.data.size())); @@ -529,8 +534,12 @@ void GetLastLineUnittest::TestGetLastLine() { void GetLastLineUnittest::TestGetLastLineEmpty() { std::string testLog = ""; - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(nullptr, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(nullptr, &ctx), + std::make_pair(nullptr, &ctx)); auto lastLine = logFileReader.GetLastLine(const_cast(testLog.data()), testLog.size()); APSARA_TEST_EQUAL_FATAL(0, int(lastLine.data.size())); APSARA_TEST_EQUAL_FATAL("", std::string(lastLine.data.data(), lastLine.data.size())); @@ -571,8 +580,12 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(nullptr, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -638,8 +651,12 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom config["StartPattern"] = LOG_BEGIN_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(nullptr, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -824,8 +841,12 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(nullptr, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -1024,8 +1045,12 @@ void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncomplet config["StartPattern"] = LOG_BEGIN_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(nullptr, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(0); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -1184,8 +1209,12 @@ void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncomplet config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(nullptr, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(0); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); diff --git a/core/unittest/sender/SenderUnittest.cpp b/core/unittest/sender/SenderUnittest.cpp index 12875eb95c..1cd6d79fe2 100644 --- a/core/unittest/sender/SenderUnittest.cpp +++ b/core/unittest/sender/SenderUnittest.cpp @@ -2684,72 +2684,6 @@ class SenderUnittest : public ::testing::Test { LOG_INFO(sLogger, ("TestTooOldFilesIntegrity() end", time(NULL))); } - void TestMergeTruncateInfo() { - LOG_INFO(sLogger, ("TestMergeTruncateInfo() begin", time(NULL))); - - std::string truInfo_1("[000-111]"); - std::string truInfo_2("[222-333]"); - std::string truInfo_3("[444-555]"); - std::string truInfo_4("[666-777]"); - std::string truInfo_5("[888-999]"); - - std::string truInfo; - truInfo.append(truInfo_1); - truInfo.append("," + truInfo_2); - truInfo.append("," + truInfo_3); - truInfo.append("," + truInfo_4); - truInfo.append("," + truInfo_5); - - MergeItem mergeItem(std::string("test_project"), - std::string("test_config_name"), - std::string("test_filename"), - true, - std::string("test_aliuid"), - std::string("test_region"), - 123456, - FlusherSLS::Batch::MergeType::LOGSTORE, - std::string("test_shardhashkey"), - 123456); - - sls_logs::LogGroup logGroup_1; - sls_logs::LogTag* logTag_1 = logGroup_1.add_logtags(); - logTag_1->set_key(LOG_RESERVED_KEY_TRUNCATE_INFO); - logTag_1->set_value(truInfo_1); - Aggregator::GetInstance()->MergeTruncateInfo(logGroup_1, &mergeItem); - - sls_logs::LogGroup logGroup_2; - sls_logs::LogTag* logTag_2 = logGroup_2.add_logtags(); - logTag_2->set_key(LOG_RESERVED_KEY_TRUNCATE_INFO); - logTag_2->set_value(truInfo_2); - Aggregator::GetInstance()->MergeTruncateInfo(logGroup_2, &mergeItem); - - sls_logs::LogGroup logGroup_3; - sls_logs::LogTag* logTag_3 = logGroup_3.add_logtags(); - logTag_3->set_key(LOG_RESERVED_KEY_TRUNCATE_INFO); - logTag_3->set_value(truInfo_3); - Aggregator::GetInstance()->MergeTruncateInfo(logGroup_3, &mergeItem); - - sls_logs::LogGroup logGroup_4; - sls_logs::LogTag* logTag_4 = logGroup_4.add_logtags(); - logTag_4->set_key(LOG_RESERVED_KEY_TRUNCATE_INFO); - logTag_4->set_value(truInfo_4); - Aggregator::GetInstance()->MergeTruncateInfo(logGroup_4, &mergeItem); - - sls_logs::LogGroup logGroup_5; - sls_logs::LogTag* logTag_5 = logGroup_5.add_logtags(); - logTag_5->set_key(LOG_RESERVED_KEY_TRUNCATE_INFO); - logTag_5->set_value(truInfo_5); - Aggregator::GetInstance()->MergeTruncateInfo(logGroup_5, &mergeItem); - - APSARA_TEST_EQUAL(mergeItem.mLogGroup.logtags_size(), 1); - - const sls_logs::LogTag& logTag = mergeItem.mLogGroup.logtags(0); - APSARA_TEST_EQUAL(logTag.key(), LOG_RESERVED_KEY_TRUNCATE_INFO); - APSARA_TEST_EQUAL(logTag.value(), truInfo); - - LOG_INFO(sLogger, ("TestMergeTruncateInfo() end", time(NULL))); - } - void TestGlobalMarkOffset() { LOG_INFO(sLogger, ("TestGlobalMarkOffset() begin", time(NULL))); // prepare diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index 215a799967..8f91bbb5b8 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -22,8 +22,6 @@ using namespace std; namespace logtail { -const string TOPIC_VALUE = "topic"; - class SLSSerializerUnittest : public ::testing::Test { public: void TestSerializeEventGroup(); @@ -380,9 +378,9 @@ void SLSSerializerUnittest::TestSerializeEventGroupList() { BatchedEvents SLSSerializerUnittest::CreateBatchedLogEvents(bool enableNanosecond, bool emptyContent) { PipelineEventGroup group(make_shared()); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("source")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("machine_uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "machine_uuid"); group.SetTag(LOG_RESERVED_KEY_PACKAGE_ID, "pack_id"); StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); @@ -397,7 +395,6 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedLogEvents(bool enableNanosecon e->SetTimestamp(1234567890); } BatchedEvents batch(std::move(group.MutableEvents()), - std::move(group.GetAllMetadata()), std::move(group.GetSizedTags()), std::move(group.GetSourceBuffer()), group.GetMetadata(EventGroupMetaKey::SOURCE_ID), @@ -411,10 +408,9 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedMetricEvents(bool enableNanose bool emptyValue, bool onlyOneTag) { PipelineEventGroup group(make_shared()); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); - group.SetMetadata(EventGroupMetaKey::SOURCE, string("127.0.0.1")); - group.SetMetadata(EventGroupMetaKey::MACHINE_UUID, string("uuid")); - group.SetMetadata(EventGroupMetaKey::TOPIC, TOPIC_VALUE); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "machine_uuid"); group.SetTag(LOG_RESERVED_KEY_PACKAGE_ID, "pack_id"); StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); @@ -437,7 +433,6 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedMetricEvents(bool enableNanose } e->SetName("test_gauge"); BatchedEvents batch(std::move(group.MutableEvents()), - std::move(group.GetAllMetadata()), std::move(group.GetSizedTags()), std::move(group.GetSourceBuffer()), group.GetMetadata(EventGroupMetaKey::SOURCE_ID), diff --git a/core/unittest/serializer/SerializerUnittest.cpp b/core/unittest/serializer/SerializerUnittest.cpp index b2bcac4b21..f5aca69d04 100644 --- a/core/unittest/serializer/SerializerUnittest.cpp +++ b/core/unittest/serializer/SerializerUnittest.cpp @@ -100,7 +100,6 @@ BatchedEvents SerializerUnittest::CreateBatchedMetricEvents(bool withEvents) { group.AddLogEvent(); } BatchedEvents batch(std::move(group.MutableEvents()), - std::move(group.GetAllMetadata()), std::move(group.GetSizedTags()), std::move(group.GetSourceBuffer()), group.GetMetadata(EventGroupMetaKey::SOURCE_ID), diff --git a/docs/cn/configuration/collection-config.md b/docs/cn/configuration/collection-config.md index 3ffadad1b0..7f2d059b4c 100644 --- a/docs/cn/configuration/collection-config.md +++ b/docs/cn/configuration/collection-config.md @@ -58,5 +58,7 @@ flushers: * 表1：Tag配置项以及默认值 | **配置项** | **是否默认添加** | **默认值** | | --- | --- | --- | -| HOST_NAME | 是 | host.name | -| HOST_IP | 是 | host.ip | +| HOST_NAME | 是 | host_name | +| HOST_IP | 是 | host_ip | +| HOST_ID | 是 | host_id | +| CLOUD_PROVIDER | 是 | cloud_provider | diff --git a/docs/cn/plugins/input/native/input-file.md b/docs/cn/plugins/input/native/input-file.md index e47c8bbd43..549a464c9b 100644 --- a/docs/cn/plugins/input/native/input-file.md +++ b/docs/cn/plugins/input/native/input-file.md @@ -58,14 +58,14 @@ * 表3：Tag配置项以及默认值 | **配置项** | **是否默认添加** | **默认值** | | --- | --- | --- | -| FileInodeTagKey | 否 | log.file.inode | -| FilePathTagKey | 是 | log.file.path | -| K8sNamespaceTagKey | 是（当EnableContainerDiscovery为true时） | k8s.namespace.name | -| K8sPodNameTagKey | 是（当EnableContainerDiscovery为true时） | k8s.pod.name | -| K8sPodUidTagKey | 是（当EnableContainerDiscovery为true时） | k8s.pod.uid | -| ContainerNameTagKey | 是（当EnableContainerDiscovery为true时） | 普通：container.name，K8s：k8s.container.name | -| ContainerIpTagKey | 是（当EnableContainerDiscovery为true时） | 普通：container.ip，K8s：k8s.container.ip | -| ContainerImageTagKey | 是（当EnableContainerDiscovery为true时） | 普通：container.image，K8s：k8s.container.image | +| FileInodeTagKey | 否 | file_offset | +| FilePathTagKey | 是 | file_path | +| K8sNamespaceTagKey | 是（当EnableContainerDiscovery为true时） | namespace | +| K8sPodNameTagKey | 是（当EnableContainerDiscovery为true时） | pod_name | +| K8sPodUidTagKey | 是（当EnableContainerDiscovery为true时） | pod_uid | +| ContainerNameTagKey | 是（当EnableContainerDiscovery为true时） | container_name | +| ContainerIpTagKey | 是（当EnableContainerDiscovery为true时） | container_ip | +| ContainerImageTagKey | 是（当EnableContainerDiscovery为true时） | image_name | ## 样例 diff --git a/pkg/config/global_config.go b/pkg/config/global_config.go index 7298e313b9..9ac35fda59 100644 --- a/pkg/config/global_config.go +++ b/pkg/config/global_config.go @@ -27,7 +27,6 @@ type GlobalConfig struct { FlushIntervalMs int DefaultLogQueueSize int DefaultLogGroupQueueSize int - Tags map[string]string // Directory to store prometheus configuration file. LoongcollectorPrometheusAuthorizationPath string // Directory to store loongcollector data, such as checkpoint, etc. diff --git a/pkg/helper/docker_center.go b/pkg/helper/docker_center.go index b81b9da393..5a3f24e689 100644 --- a/pkg/helper/docker_center.go +++ b/pkg/helper/docker_center.go @@ -569,6 +569,7 @@ func (dc *DockerCenter) CreateInfoDetail(info types.ContainerJSON, envConfigPref k8sInfo := K8SInfo{} ip := dc.getIPAddress(info) + containerNameTag["_image_name_"] = dc.getImageName(info.Image, info.Config.Image) if strings.HasPrefix(info.Name, "/k8s_") || strings.HasPrefix(info.Name, "k8s_") || strings.Count(info.Name, "_") >= 4 { // 1. container name is k8s // k8s_php-redis_frontend-2337258262-154p7_default_d8a2e2dd-3617-11e7-a4b0-ecf4bbe5d414_0 @@ -587,8 +588,7 @@ func (dc *DockerCenter) CreateInfoDetail(info types.ContainerJSON, envConfigPref if len(tags) == 6 { baseIndex = 1 } - containerNameTag["_k8s_image_name_"] = dc.getImageName(info.Image, info.Config.Image) - containerNameTag["_k8s_container_name_"] = tags[baseIndex] + containerNameTag["_container_name_"] = tags[baseIndex] containerNameTag["_pod_name_"] = tags[baseIndex+1] containerNameTag["_namespace_"] = tags[baseIndex+2] containerNameTag["_pod_uid_"] = tags[baseIndex+3] @@ -596,13 +596,9 @@ func (dc *DockerCenter) CreateInfoDetail(info types.ContainerJSON, envConfigPref k8sInfo.Pod = tags[baseIndex+1] k8sInfo.Namespace = tags[baseIndex+2] k8sInfo.ExtractK8sLabels(info) - if len(ip) > 0 { - containerNameTag["_k8s_container_ip_"] = ip - } } else if _, ok := info.Config.Labels[k8sPodNameLabel]; ok { // 2. container labels has k8sPodNameLabel - containerNameTag["_k8s_image_name_"] = dc.getImageName(info.Image, info.Config.Image) - containerNameTag["_k8s_container_name_"] = info.Name + containerNameTag["_container_name_"] = info.Name containerNameTag["_pod_name_"] = info.Config.Labels[k8sPodNameLabel] containerNameTag["_namespace_"] = info.Config.Labels[k8sPodNameSpaceLabel] containerNameTag["_pod_uid_"] = info.Config.Labels[k8sPodUUIDLabel] @@ -612,20 +608,13 @@ func (dc *DockerCenter) CreateInfoDetail(info types.ContainerJSON, envConfigPref // the following method is couped with the CRI adapter, only the original docker container labels // would be added to the labels of the k8s info. k8sInfo.ExtractK8sLabels(info) - if len(ip) > 0 { - containerNameTag["_k8s_container_ip_"] = ip - } } else { // 3. treat as normal container - containerNameTag["_image_name_"] = dc.getImageName(info.Image, info.Config.Image) if strings.HasPrefix(info.Name, "/") { containerNameTag["_container_name_"] = info.Name[1:] } else { containerNameTag["_container_name_"] = info.Name } - if len(ip) > 0 { - containerNameTag["_container_ip_"] = ip - } } did := &DockerInfoDetail{ diff --git a/pkg/protocol/converter/converter.go b/pkg/protocol/converter/converter.go index af81369738..53d0835e18 100644 --- a/pkg/protocol/converter/converter.go +++ b/pkg/protocol/converter/converter.go @@ -25,25 +25,6 @@ import ( "github.com/alibaba/ilogtail/pkg/protocol" ) -const ( - tagHostIP = "host.ip" - tagLogTopic = "log.topic" - tagLogFilePath = "log.file.path" - tagHostname = "host.name" - tagK8sNodeIP = "k8s.node.ip" - tagK8sNodeName = "k8s.node.name" - tagK8sNamespace = "k8s.namespace.name" - tagK8sPodName = "k8s.pod.name" - tagK8sPodIP = "k8s.pod.ip" - tagK8sPodUID = "k8s.pod.uid" - tagContainerName = "container.name" - tagContainerIP = "container.ip" - tagContainerImageName = "container.image.name" - tagK8sContainerName = "k8s.container.name" - tagK8sContainerIP = "k8s.container.ip" - tagK8sContainerImageName = "k8s.container.image.name" -) - const ( ProtocolCustomSingle = "custom_single" ProtocolCustomSingleFlatten = "custom_single_flatten" @@ -68,6 +49,25 @@ const ( targetGroupMetadataPrefix = "metadata." ) +const ( + tagHostIP = "host.ip" + tagLogTopic = "log.topic" + tagLogFilePath = "log.file.path" + tagHostname = "host.name" + tagK8sNodeIP = "k8s.node.ip" + tagK8sNodeName = "k8s.node.name" + tagK8sNamespace = "k8s.namespace.name" + tagK8sPodName = "k8s.pod.name" + tagK8sPodIP = "k8s.pod.ip" + tagK8sPodUID = "k8s.pod.uid" + tagContainerName = "container.name" + tagContainerIP = "container.ip" + tagContainerImageName = "container.image.name" + tagK8sContainerName = "k8s.container.name" + tagK8sContainerIP = "k8s.container.ip" + tagK8sContainerImageName = "k8s.container.image.name" +) + // todo: make multiple pools for different size levels var byteBufPool = sync.Pool{ New: func() interface{} { diff --git a/pluginmanager/plugin_runner_helper.go b/pluginmanager/plugin_runner_helper.go index 9b8d2390eb..37375cd229 100644 --- a/pluginmanager/plugin_runner_helper.go +++ b/pluginmanager/plugin_runner_helper.go @@ -20,10 +20,7 @@ import ( "strings" "time" - "github.com/alibaba/ilogtail/pkg/config" - "github.com/alibaba/ilogtail/pkg/helper" "github.com/alibaba/ilogtail/pkg/logger" - "github.com/alibaba/ilogtail/pkg/models" "github.com/alibaba/ilogtail/pkg/pipeline" "github.com/alibaba/ilogtail/pkg/util" ) @@ -95,17 +92,6 @@ func flushOutStore[T FlushData, F FlusherWrapperInterface](lc *LogstoreConfig, s return true } -func loadAdditionalTags(globalConfig *config.GlobalConfig) models.Tags { - tags := models.NewTagsWithKeyValues() - for i := 0; i < len(helper.EnvTags); i += 2 { - tags.Add(helper.EnvTags[i], helper.EnvTags[i+1]) - } - for key, value := range globalConfig.Tags { - tags.Add(key, value) - } - return tags -} - func GetFlushStoreLen(runner PluginRunner) int { if r, ok := runner.(*pluginv1Runner); ok { return r.FlushOutStore.Len() diff --git a/pluginmanager/processor_tag_commuinty.go b/pluginmanager/processor_tag_commuinty.go index 4ad838048f..728c1d6a7f 100644 --- a/pluginmanager/processor_tag_commuinty.go +++ b/pluginmanager/processor_tag_commuinty.go @@ -17,16 +17,18 @@ package pluginmanager import ( - "github.com/alibaba/ilogtail/pkg/config" + "github.com/alibaba/ilogtail/pkg/helper" "github.com/alibaba/ilogtail/pkg/models" "github.com/alibaba/ilogtail/pkg/pipeline" "github.com/alibaba/ilogtail/pkg/util" ) const ( - hostNameDefaultTagKey = "host.name" - hostIPDefaultTagKey = "host.ip" - defaultConfigTagKeyValue = "__default__" + hostNameDefaultTagKey = "host_name" + hostIPDefaultTagKey = "host_ip" + hostIDDefaultTagKey = "host_id" + cloudProviderDefaultTagKey = "cloud_provider" + defaultConfigTagKeyValue = "__default__" ) // Processor interface cannot meet the requirements of tag processing, so we need to create a special ProcessorTag struct @@ -36,7 +38,7 @@ type ProcessorTag struct { AgentEnvMetaTagKey map[string]string } -func (p *ProcessorTag) ProcessV1(logCtx *pipeline.LogWithContext, globalConfig *config.GlobalConfig) { +func (p *ProcessorTag) ProcessV1(logCtx *pipeline.LogWithContext) { tags, ok := logCtx.Context["tags"] if !ok { return @@ -45,32 +47,32 @@ func (p *ProcessorTag) ProcessV1(logCtx *pipeline.LogWithContext, globalConfig * if !ok { return } - p.addTagIfRequired("HOST_NAME", hostNameDefaultTagKey, util.GetHostName(), tagsMap) - p.addTagIfRequired("HOST_IP", hostIPDefaultTagKey, util.GetIPAddress(), tagsMap) + p.addDefaultAddedTag("HOST_NAME", hostNameDefaultTagKey, util.GetHostName(), tagsMap) + p.addDefaultAddedTag("HOST_IP", hostIPDefaultTagKey, util.GetIPAddress(), tagsMap) // Add tags for each log, includes: default hostname tag, - // env tags and global tags in config. - for k, v := range loadAdditionalTags(globalConfig).Iterator() { - tagsMap[k] = v + // env tags + for i := 0; i < len(helper.EnvTags); i += 2 { + tagsMap[helper.EnvTags[i]] = helper.EnvTags[i+1] } } -func (p *ProcessorTag) ProcessV2(in *models.PipelineGroupEvents, globalConfig *config.GlobalConfig) { +func (p *ProcessorTag) ProcessV2(in *models.PipelineGroupEvents) { tagsMap := make(map[string]string) - p.addTagIfRequired("HOST_NAME", hostNameDefaultTagKey, util.GetHostName(), tagsMap) - p.addTagIfRequired("HOST_IP", hostIPDefaultTagKey, util.GetIPAddress(), tagsMap) + p.addDefaultAddedTag("HOST_NAME", hostNameDefaultTagKey, util.GetHostName(), tagsMap) + p.addDefaultAddedTag("HOST_IP", hostIPDefaultTagKey, util.GetIPAddress(), tagsMap) for k, v := range tagsMap { in.Group.Tags.Add(k, v) } // Add tags for each log, includes: default hostname tag, - // env tags and global tags in config. - for k, v := range loadAdditionalTags(globalConfig).Iterator() { - in.Group.Tags.Add(k, v) + // env tags + for i := 0; i < len(helper.EnvTags); i += 2 { + in.Group.Tags.Add(helper.EnvTags[i], helper.EnvTags[i+1]) } } -func (p *ProcessorTag) addTagIfRequired(configKey, defaultKey, value string, tags map[string]string) { +func (p *ProcessorTag) addDefaultAddedTag(configKey, defaultKey, value string, tags map[string]string) { if key, ok := p.PipelineMetaTagKey[configKey]; ok { if key != "" { if key == defaultConfigTagKeyValue { @@ -79,7 +81,21 @@ func (p *ProcessorTag) addTagIfRequired(configKey, defaultKey, value string, tag tags[key] = value } } + // emtpy value means delete } else { tags[defaultKey] = value } } + +func (p *ProcessorTag) addOptionalTag(configKey, defaultKey, value string, tags map[string]string) { + if key, ok := p.PipelineMetaTagKey[configKey]; ok { + if key != "" { + if key == defaultConfigTagKeyValue { + tags[defaultKey] = value + } else { + tags[key] = value + } + } + // emtpy value means delete + } +} diff --git a/pluginmanager/processor_tag_community_test.go b/pluginmanager/processor_tag_community_test.go old mode 100644 new mode 100755 index c1932f09fa..ff314d3817 --- a/pluginmanager/processor_tag_community_test.go +++ b/pluginmanager/processor_tag_community_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/alibaba/ilogtail/pkg/config" "github.com/alibaba/ilogtail/pkg/helper" "github.com/alibaba/ilogtail/pkg/models" "github.com/alibaba/ilogtail/pkg/pipeline" @@ -36,19 +35,17 @@ func TestTagDefault(t *testing.T) { processorTag := ProcessorTag{ PipelineMetaTagKey: map[string]string{}, EnableAgentEnvMetaTagControl: false, - AgentEnvMetaTagKey: map[string]string{}, } logCtx := &pipeline.LogWithContext{ Context: map[string]interface{}{ "tags": map[string]string{}, }, } - globalConfig := &config.GlobalConfig{} - processorTag.ProcessV1(logCtx, globalConfig) + processorTag.ProcessV1(logCtx) tagsMap := logCtx.Context["tags"].(map[string]string) assert.Equal(t, 3, len(tagsMap)) - assert.Equal(t, util.GetHostName(), tagsMap["host.name"]) - assert.Equal(t, util.GetIPAddress(), tagsMap["host.ip"]) + assert.Equal(t, util.GetHostName(), tagsMap[hostNameDefaultTagKey]) + assert.Equal(t, util.GetIPAddress(), tagsMap[hostIPDefaultTagKey]) assert.Equal(t, "test_env_tag_value", tagsMap["test_env_tag"]) processorTag.PipelineMetaTagKey = map[string]string{ @@ -60,11 +57,11 @@ func TestTagDefault(t *testing.T) { "tags": map[string]string{}, }, } - processorTag.ProcessV1(logCtx, globalConfig) + processorTag.ProcessV1(logCtx) tagsMap = logCtx.Context["tags"].(map[string]string) assert.Equal(t, 3, len(tagsMap)) - assert.Equal(t, util.GetHostName(), tagsMap["host.name"]) - assert.Equal(t, util.GetIPAddress(), tagsMap["host.ip"]) + assert.Equal(t, util.GetHostName(), tagsMap[hostNameDefaultTagKey]) + assert.Equal(t, util.GetIPAddress(), tagsMap[hostIPDefaultTagKey]) assert.Equal(t, "test_env_tag_value", tagsMap["test_env_tag"]) } @@ -76,17 +73,15 @@ func TestTagDefaultV2(t *testing.T) { processorTag := ProcessorTag{ PipelineMetaTagKey: map[string]string{}, EnableAgentEnvMetaTagControl: false, - AgentEnvMetaTagKey: map[string]string{}, } in := &models.PipelineGroupEvents{ Group: &models.GroupInfo{ Tags: models.NewTags(), }, } - globalConfig := &config.GlobalConfig{} - processorTag.ProcessV2(in, globalConfig) - assert.Equal(t, util.GetHostName(), in.Group.Tags.Get("host.name")) - assert.Equal(t, util.GetIPAddress(), in.Group.Tags.Get("host.ip")) + processorTag.ProcessV2(in) + assert.Equal(t, util.GetHostName(), in.Group.Tags.Get(hostNameDefaultTagKey)) + assert.Equal(t, util.GetIPAddress(), in.Group.Tags.Get(hostIPDefaultTagKey)) assert.Equal(t, "test_env_tag_value", in.Group.Tags.Get("test_env_tag")) processorTag.PipelineMetaTagKey = map[string]string{ @@ -98,9 +93,9 @@ func TestTagDefaultV2(t *testing.T) { Tags: models.NewTags(), }, } - processorTag.ProcessV2(in, globalConfig) - assert.Equal(t, util.GetHostName(), in.Group.Tags.Get("host.name")) - assert.Equal(t, util.GetIPAddress(), in.Group.Tags.Get("host.ip")) + processorTag.ProcessV2(in) + assert.Equal(t, util.GetHostName(), in.Group.Tags.Get(hostNameDefaultTagKey)) + assert.Equal(t, util.GetIPAddress(), in.Group.Tags.Get(hostIPDefaultTagKey)) assert.Equal(t, "test_env_tag_value", in.Group.Tags.Get("test_env_tag")) } @@ -114,16 +109,13 @@ func TestTagRename(t *testing.T) { "HOST_NAME": "test_host_name", "HOST_IP": "test_host_ip", }, - EnableAgentEnvMetaTagControl: false, - AgentEnvMetaTagKey: map[string]string{}, } logCtx := &pipeline.LogWithContext{ Context: map[string]interface{}{ "tags": map[string]string{}, }, } - globalConfig := &config.GlobalConfig{} - processorTag.ProcessV1(logCtx, globalConfig) + processorTag.ProcessV1(logCtx) tagsMap := logCtx.Context["tags"].(map[string]string) assert.Equal(t, 3, len(tagsMap)) assert.Equal(t, util.GetHostName(), tagsMap["test_host_name"]) @@ -141,16 +133,13 @@ func TestTagRenameV2(t *testing.T) { "HOST_NAME": "test_host_name", "HOST_IP": "test_host_ip", }, - EnableAgentEnvMetaTagControl: false, - AgentEnvMetaTagKey: map[string]string{}, } in := &models.PipelineGroupEvents{ Group: &models.GroupInfo{ Tags: models.NewTags(), }, } - globalConfig := &config.GlobalConfig{} - processorTag.ProcessV2(in, globalConfig) + processorTag.ProcessV2(in) assert.Equal(t, util.GetHostName(), in.Group.Tags.Get("test_host_name")) assert.Equal(t, util.GetIPAddress(), in.Group.Tags.Get("test_host_ip")) } @@ -165,16 +154,13 @@ func TestTagDelete(t *testing.T) { "HOST_NAME": "", "HOST_IP": "", }, - EnableAgentEnvMetaTagControl: false, - AgentEnvMetaTagKey: map[string]string{}, } logCtx := &pipeline.LogWithContext{ Context: map[string]interface{}{ "tags": map[string]string{}, }, } - globalConfig := &config.GlobalConfig{} - processorTag.ProcessV1(logCtx, globalConfig) + processorTag.ProcessV1(logCtx) tagsMap := logCtx.Context["tags"].(map[string]string) assert.Equal(t, 1, len(tagsMap)) assert.Equal(t, "test_env_tag_value", tagsMap["test_env_tag"]) @@ -190,16 +176,13 @@ func TestTagDeleteV2(t *testing.T) { "HOST_NAME": "", "HOST_IP": "", }, - EnableAgentEnvMetaTagControl: false, - AgentEnvMetaTagKey: map[string]string{}, } in := &models.PipelineGroupEvents{ Group: &models.GroupInfo{ Tags: models.NewTags(), }, } - globalConfig := &config.GlobalConfig{} - processorTag.ProcessV2(in, globalConfig) + processorTag.ProcessV2(in) assert.Equal(t, "", in.Group.Tags.Get("HOST_NAME")) assert.Equal(t, "", in.Group.Tags.Get("HOST_IP")) assert.Equal(t, "test_env_tag_value", in.Group.Tags.Get("test_env_tag"))