From e09680547bfbd848b0597a44f7054b0d2d095814 Mon Sep 17 00:00:00 2001 From: zhb192729 Date: Thu, 19 Dec 2024 09:39:00 +0800 Subject: [PATCH 1/3] fix ut --- core/unittest/metadata/K8sMetadataUnittest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/metadata/K8sMetadataUnittest.cpp b/core/unittest/metadata/K8sMetadataUnittest.cpp index 46b077e186..b6cc07b5ef 100644 --- a/core/unittest/metadata/K8sMetadataUnittest.cpp +++ b/core/unittest/metadata/K8sMetadataUnittest.cpp @@ -259,7 +259,7 @@ class k8sMetadataUnittest : public ::testing::Test { auto& k8sMetadata = K8sMetadata::GetInstance(); k8sMetadata.SetIpCache(root); - k8sMetadata.GetByLocalHostFromServer(); + //k8sMetadata.GetByLocalHostFromServer(); unique_ptr mSpanEvent; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); From f68b24aab3a1aaf16180a1c17103521537918168 Mon Sep 17 00:00:00 2001 From: zhb192729 Date: Thu, 19 Dec 2024 15:09:31 +0800 Subject: [PATCH 2/3] fix test --- core/metadata/K8sMetadata.cpp | 5 +++++ core/metadata/K8sMetadata.h | 10 +++++++--- core/metadata/LabelingK8sMetadata.cpp | 12 ++++++------ core/unittest/metadata/CMakeLists.txt | 6 +++--- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/core/metadata/K8sMetadata.cpp b/core/metadata/K8sMetadata.cpp index 3829e3fb91..2dfdbf55e2 100644 --- a/core/metadata/K8sMetadata.cpp +++ b/core/metadata/K8sMetadata.cpp @@ -93,6 +93,11 @@ bool K8sMetadata::FromContainerJson(const Json::Value& json, std::shared_ptr data); - public: - static K8sMetadata& GetInstance() { + // Destructor + ~K8sMetadata() { + ClearCache(); + } + static K8sMetadata* GetInstance() { static K8sMetadata instance(500); - return instance; + return &instance; } // 公共方法 //if cache not have,get from server @@ -87,6 +90,7 @@ namespace logtail { // get info by ip from cache std::shared_ptr GetInfoByIpFromCache(const std::string& ip); bool SendRequestToOperator(const std::string& urlHost, const std::string& output, containerInfoType infoType); + void ClearCache(); #ifdef APSARA_UNIT_TEST_MAIN friend class k8sMetadataUnittest; diff --git a/core/metadata/LabelingK8sMetadata.cpp b/core/metadata/LabelingK8sMetadata.cpp index 57eee58771..b6f1c04dd0 100644 --- a/core/metadata/LabelingK8sMetadata.cpp +++ b/core/metadata/LabelingK8sMetadata.cpp @@ -43,11 +43,11 @@ void LabelingK8sMetadata::AddLabelToLogGroup(PipelineEventGroup& logGroup) { cotainerNotTag.push_back(rIdx); } } - auto& k8sMetadata = K8sMetadata::GetInstance(); - if (containerVec.empty() || (!k8sMetadata.GetByContainerIdsFromServer(containerVec))) { + K8sMetadata* k8sMetadata = K8sMetadata::GetInstance(); + if (containerVec.empty() || (!k8sMetadata->GetByContainerIdsFromServer(containerVec))) { return; } - if (remoteIpVec.empty() || (!k8sMetadata.GetByIpsFromServer(remoteIpVec))) { + if (remoteIpVec.empty() || (!k8sMetadata->GetByIpsFromServer(remoteIpVec))) { return; } for (size_t i = 0; i < cotainerNotTag.size(); ++i) { @@ -74,12 +74,12 @@ template bool LabelingK8sMetadata::AddLabels(Event& e, std::vector& containerVec, std::vector& remoteIpVec) { bool res = true; - auto& k8sMetadata = K8sMetadata::GetInstance(); + K8sMetadata* k8sMetadata = K8sMetadata::GetInstance(); StringView containerIdViewKey(containerIdKey); StringView containerIdView = e.HasTag(containerIdViewKey) ? e.GetTag(containerIdViewKey) : StringView{}; if (!containerIdView.empty()) { std::string containerId(containerIdView); - std::shared_ptr containerInfo = k8sMetadata.GetInfoByContainerIdFromCache(containerId); + std::shared_ptr containerInfo = k8sMetadata->GetInfoByContainerIdFromCache(containerId); if (containerInfo == nullptr) { containerVec.push_back(containerId); res = false; @@ -95,7 +95,7 @@ bool LabelingK8sMetadata::AddLabels(Event& e, std::vector& containe StringView remoteIpView = e.HasTag(ipView) ? e.GetTag(ipView) : StringView{}; if (!remoteIpView.empty()) { std::string remoteIp(remoteIpView); - std::shared_ptr ipInfo = k8sMetadata.GetInfoByIpFromCache(remoteIp); + std::shared_ptr ipInfo = k8sMetadata->GetInfoByIpFromCache(remoteIp); if (ipInfo == nullptr) { remoteIpVec.push_back(remoteIp); res = false; diff --git a/core/unittest/metadata/CMakeLists.txt b/core/unittest/metadata/CMakeLists.txt index fca663ca07..1a52c8348f 100644 --- a/core/unittest/metadata/CMakeLists.txt +++ b/core/unittest/metadata/CMakeLists.txt @@ -15,8 +15,8 @@ cmake_minimum_required(VERSION 3.22) project(metadata_unittest) -# add_executable(metadata_unittest K8sMetadataUnittest.cpp) -# target_link_libraries(metadata_unittest ${UT_BASE_TARGET}) + add_executable(metadata_unittest K8sMetadataUnittest.cpp) + target_link_libraries(metadata_unittest ${UT_BASE_TARGET}) include(GoogleTest) -# gtest_discover_tests(metadata_unittest) \ No newline at end of file + gtest_discover_tests(metadata_unittest) \ No newline at end of file From 04d6343d00297ba55041af534d326f05b0b71a51 Mon Sep 17 00:00:00 2001 From: zhb192729 Date: Thu, 19 Dec 2024 15:34:16 +0800 Subject: [PATCH 3/3] fix test --- .../unittest/metadata/K8sMetadataUnittest.cpp | 270 +++--------------- 1 file changed, 45 insertions(+), 225 deletions(-) diff --git a/core/unittest/metadata/K8sMetadataUnittest.cpp b/core/unittest/metadata/K8sMetadataUnittest.cpp index b6cc07b5ef..06264d6128 100644 --- a/core/unittest/metadata/K8sMetadataUnittest.cpp +++ b/core/unittest/metadata/K8sMetadataUnittest.cpp @@ -1,17 +1,3 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - #include "unittest/Unittest.h" #include #include @@ -19,49 +5,20 @@ #include "metadata/LabelingK8sMetadata.h" #include "metadata/K8sMetadata.h" #include "models/PipelineEventGroup.h" +#include using namespace std; +DECLARE_FLAG_STRING(ilogtail_config); + namespace logtail { + class k8sMetadataUnittest : public ::testing::Test { protected: void SetUp() override { - // You can set up common objects needed for each test case here - } - - void TearDown() override { - // Clean up after each test case if needed - } - -public: - void TestGetByContainerIds() { - LOG_INFO(sLogger, ("TestGetByContainerIds() begin", time(NULL))); - const std::string jsonData = R"({"containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144":{"namespace":"default","workloadName":"oneagent-demo-658648895b","workloadKind":"replicaset","serviceName":"","labels":{"app":"oneagent-demo","pod-template-hash":"658648895b"},"envs":{},"images":{"oneagent-demo":"sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/centos7-cve-fix:1.0.0"}}})"; - - Json::Value root; - Json::CharReaderBuilder readerBuilder; - std::istringstream jsonStream(jsonData); - std::string errors; - - // Parse JSON data - if (!Json::parseFromStream(readerBuilder, jsonStream, &root, &errors)) { - std::cerr << "Failed to parse JSON: " << errors << std::endl; - return; - } - auto& k8sMetadata = K8sMetadata::GetInstance(); - k8sMetadata.SetContainerCache(root); - k8sMetadata.GetByLocalHostFromServer(); - - // Assume GetInfoByContainerIdFromCache returns non-null shared_ptr for valid IDs, - // and check for some expectations. - APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByContainerIdFromCache("containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144") != nullptr); - - } - - void TestGetByLocalHost() { - LOG_INFO(sLogger, ("TestGetByLocalHost() begin", time(NULL))); - // Sample JSON data + // Common setup for k8s metadata const std::string jsonData = R"({ + "containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144":{"namespace":"default","workloadName":"oneagent-demo-658648895b","workloadKind":"replicaset","serviceName":"","labels":{"app":"oneagent-demo","pod-template-hash":"658648895b"},"envs":{},"images":{"oneagent-demo":"sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/centos7-cve-fix:1.0.0"}}, "10.41.0.2": { "namespace": "kube-system", "workloadName": "coredns-7b669cbb96", @@ -125,20 +82,43 @@ class k8sMetadataUnittest : public ::testing::Test { } })"; - Json::Value root; Json::CharReaderBuilder readerBuilder; std::istringstream jsonStream(jsonData); std::string errors; // Parse JSON data - if (!Json::parseFromStream(readerBuilder, jsonStream, &root, &errors)) { + if (!Json::parseFromStream(readerBuilder, jsonStream, &parsedJsonData, &errors)) { std::cerr << "Failed to parse JSON: " << errors << std::endl; return; } + k8sMetadata = K8sMetadata::GetInstance(); + k8sMetadata->SetContainerCache(parsedJsonData); + k8sMetadata->SetIpCache(parsedJsonData); + + } + + void TearDown() override { + // Clear the data cache for K8sMetadata to speed up the recovery time between tests. + //auto& k8sMetadata = K8sMetadata::GetInstance(); + k8sMetadata->ClearCache(); + //delete k8sMetadata; + return; + } + +private: + Json::Value parsedJsonData; + K8sMetadata* k8sMetadata; +public: + void TestGetByContainerIds() { + LOG_INFO(sLogger, ("TestGetByContainerIds() begin", time(NULL))); + //auto& k8sMetadata = K8sMetadata::GetInstance(); + APSARA_TEST_TRUE_FATAL(k8sMetadata->GetInfoByContainerIdFromCache("containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144") != nullptr); + } - auto& k8sMetadata = K8sMetadata::GetInstance(); - k8sMetadata.SetIpCache(root); + void TestGetByLocalHost() { + LOG_INFO(sLogger, ("TestGetByLocalHost() begin", time(NULL))); + //auto& k8sMetadata = K8sMetadata::GetInstance(); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); @@ -175,91 +155,12 @@ class k8sMetadataUnittest : public ::testing::Test { EventsContainer& eventsEnd = eventGroup.MutableEvents(); auto& metricEvent = eventsEnd[0].Cast(); APSARA_TEST_EQUAL("kube-proxy-worker", metricEvent.GetTag("peerWorkloadName").to_string()); - APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr); + APSARA_TEST_TRUE_FATAL(k8sMetadata->GetInfoByIpFromCache("10.41.0.2") != nullptr); } - void TestAddLabelToSpan() { + void TestAddLabelToSpan() { LOG_INFO(sLogger, ("TestProcessEventForSpan() begin", time(NULL))); - // Sample JSON data - const std::string jsonData = R"({ - "10.41.0.2": { - "namespace": "kube-system", - "workloadName": "coredns-7b669cbb96", - "workloadKind": "replicaset", - "serviceName": "", - "labels": { - "k8s-app": "kube-dns", - "pod-template-hash": "7b669cbb96" - }, - "envs": { - "COREDNS_NAMESPACE": "", - "COREDNS_POD_NAME": "" - }, - "images": { - "coredns": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/coredns:v1.9.3.10-7dfca203-aliyun" - } - }, - "10.41.0.3": { - "namespace": "kube-system", - "workloadName": "csi-provisioner-8bd988c55", - "workloadKind": "replicaset", - "serviceName": "", - "labels": { - "app": "csi-provisioner", - "pod-template-hash": "8bd988c55" - }, - "envs": { - "CLUSTER_ID": "c33235919ddad4f279b3a67c2f0046704", - "ENABLE_NAS_SUBPATH_FINALIZER": "true", - "KUBE_NODE_NAME": "", - "SERVICE_TYPE": "provisioner" - }, - "images": { - "csi-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-plugin:v1.30.3-921e63a-aliyun", - "external-csi-snapshotter": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-snapshotter:v4.0.0-a230d5b-aliyun", - "external-disk-attacher": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-attacher:v4.5.0-4a01fda6-aliyun", - "external-disk-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-provisioner:v3.5.0-e7da67e52-aliyun", - "external-disk-resizer": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-resizer:v1.3-e48d981-aliyun", - "external-nas-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-provisioner:v3.5.0-e7da67e52-aliyun", - "external-nas-resizer": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-resizer:v1.3-e48d981-aliyun", - "external-oss-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-provisioner:v3.5.0-e7da67e52-aliyun", - "external-snapshot-controller": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/snapshot-controller:v4.0.0-a230d5b-aliyun" - } - }, - "172.16.20.108": { - "namespace": "kube-system", - "workloadName": "kube-proxy-worker", - "workloadKind": "daemonset", - "serviceName": "", - "labels": { - "controller-revision-hash": "756748b889", - "k8s-app": "kube-proxy-worker", - "pod-template-generation": "1" - }, - "envs": { - "NODE_NAME": "" - }, - "images": { - "kube-proxy-worker": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/kube-proxy:v1.30.1-aliyun.1" - } - } - })"; - - Json::Value root; - Json::CharReaderBuilder readerBuilder; - std::istringstream jsonStream(jsonData); - std::string errors; - - // Parse JSON data - if (!Json::parseFromStream(readerBuilder, jsonStream, &root, &errors)) { - std::cerr << "Failed to parse JSON: " << errors << std::endl; - return; - } - - - auto& k8sMetadata = K8sMetadata::GetInstance(); - k8sMetadata.SetIpCache(root); - //k8sMetadata.GetByLocalHostFromServer(); + //auto& k8sMetadata = K8sMetadata::GetInstance(); unique_ptr mSpanEvent; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); @@ -285,96 +186,17 @@ class k8sMetadataUnittest : public ::testing::Test { std::vector remote_ip_vec; mSpanEvent->SetStatus(SpanEvent::StatusCode::Ok); mSpanEvent->SetScopeTag(string("key2"), string("value2")); - LabelingK8sMetadata& processor = *(new LabelingK8sMetadata); + LabelingK8sMetadata processor; processor.AddLabels(*mSpanEvent, container_vec, remote_ip_vec); APSARA_TEST_EQUAL("kube-proxy-worker", mSpanEvent->GetTag("peerWorkloadName").to_string()); - APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr); + APSARA_TEST_TRUE_FATAL(k8sMetadata->GetInfoByIpFromCache("10.41.0.2") != nullptr); + delete l; + delete e; } - - void TestAddLabelToMetric() { + void TestAddLabelToMetric() { LOG_INFO(sLogger, ("TestGetByLocalHost() begin", time(NULL))); - // Sample JSON data - const std::string jsonData = R"({ - "10.41.0.2": { - "namespace": "kube-system", - "workloadName": "coredns-7b669cbb96", - "workloadKind": "replicaset", - "serviceName": "", - "labels": { - "k8s-app": "kube-dns", - "pod-template-hash": "7b669cbb96" - }, - "envs": { - "COREDNS_NAMESPACE": "", - "COREDNS_POD_NAME": "" - }, - "images": { - "coredns": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/coredns:v1.9.3.10-7dfca203-aliyun" - } - }, - "10.41.0.3": { - "namespace": "kube-system", - "workloadName": "csi-provisioner-8bd988c55", - "workloadKind": "replicaset", - "serviceName": "", - "labels": { - "app": "csi-provisioner", - "pod-template-hash": "8bd988c55" - }, - "envs": { - "CLUSTER_ID": "c33235919ddad4f279b3a67c2f0046704", - "ENABLE_NAS_SUBPATH_FINALIZER": "true", - "KUBE_NODE_NAME": "", - "SERVICE_TYPE": "provisioner" - }, - "images": { - "csi-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-plugin:v1.30.3-921e63a-aliyun", - "external-csi-snapshotter": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-snapshotter:v4.0.0-a230d5b-aliyun", - "external-disk-attacher": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-attacher:v4.5.0-4a01fda6-aliyun", - "external-disk-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-provisioner:v3.5.0-e7da67e52-aliyun", - "external-disk-resizer": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-resizer:v1.3-e48d981-aliyun", - "external-nas-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-provisioner:v3.5.0-e7da67e52-aliyun", - "external-nas-resizer": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-resizer:v1.3-e48d981-aliyun", - "external-oss-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-provisioner:v3.5.0-e7da67e52-aliyun", - "external-snapshot-controller": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/snapshot-controller:v4.0.0-a230d5b-aliyun" - } - }, - "172.16.20.108": { - "namespace": "kube-system", - "workloadName": "kube-proxy-worker", - "workloadKind": "daemonset", - "serviceName": "", - "labels": { - "controller-revision-hash": "756748b889", - "k8s-app": "kube-proxy-worker", - "pod-template-generation": "1" - }, - "envs": { - "NODE_NAME": "" - }, - "images": { - "kube-proxy-worker": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/kube-proxy:v1.30.1-aliyun.1" - } - } - })"; - - Json::Value root; - Json::CharReaderBuilder readerBuilder; - std::istringstream jsonStream(jsonData); - std::string errors; - - // Parse JSON data - if (!Json::parseFromStream(readerBuilder, jsonStream, &root, &errors)) { - std::cerr << "Failed to parse JSON: " << errors << std::endl; - return; - } - - - auto& k8sMetadata = K8sMetadata::GetInstance(); - k8sMetadata.SetIpCache(root); - k8sMetadata.GetByLocalHostFromServer(); - + // auto& k8sMetadata = K8sMetadata::GetInstance(); auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::string eventStr = R"({ @@ -405,7 +227,7 @@ class k8sMetadataUnittest : public ::testing::Test { })"; eventGroup.FromJsonString(eventStr); eventGroup.AddMetricEvent(); - LabelingK8sMetadata& processor = *(new LabelingK8sMetadata); + LabelingK8sMetadata processor; std::vector container_vec; std::vector remote_ip_vec; EventsContainer& events = eventGroup.MutableEvents(); @@ -413,7 +235,7 @@ class k8sMetadataUnittest : public ::testing::Test { EventsContainer& eventsEnd = eventGroup.MutableEvents(); auto& metricEvent = eventsEnd[0].Cast(); APSARA_TEST_EQUAL("kube-proxy-worker", metricEvent.GetTag("peerWorkloadName").to_string()); - APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr); + APSARA_TEST_TRUE_FATAL(k8sMetadata->GetInfoByIpFromCache("10.41.0.2") != nullptr); } }; @@ -422,12 +244,10 @@ APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestGetByLocalHost, 1); APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestAddLabelToMetric, 2); APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestAddLabelToSpan, 3); - - } // end of namespace logtail int main(int argc, char** argv) { logtail::Logger::Instance().InitGlobalLoggers(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); -} \ No newline at end of file +}