Remove LogFileProfiler (alibaba#1889)

* init * polish
Takuka0311 · Nov 18, 2024 · 18cc292 · 18cc292
1 parent f49f501
commit 18cc292
Show file tree

Hide file tree

Showing 29 changed files with 193 additions and 1,029 deletions.
diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp
@@ -30,7 +30,6 @@
 #include "file_server/reader/LogFileReader.h"
 #include "json/value.h"
 #include "logger/Logger.h"
-#include "monitor/LogFileProfiler.h"
 #include "monitor/AlarmManager.h"
 #include "monitor/Monitor.h"
 #ifdef __ENTERPRISE__
@@ -433,14 +432,6 @@ string GetAgentSnapshotDir() {
     }
 }
 
-string GetAgentProfileLogName() {
-    if (BOOL_FLAG(logtail_mode)) {
-        return "ilogtail_profile.LOG";
-    } else {
-        return "loongcollector_profile.LOG";
-    }
-}
-
 string GetAgentStatusLogName() {
     if (BOOL_FLAG(logtail_mode)) {
         return "ilogtail_status.LOG";
@@ -449,15 +440,6 @@ string GetAgentStatusLogName() {
     }
 }
 
-string GetProfileSnapshotDumpFileName() {
-    if (BOOL_FLAG(logtail_mode)) {
-        return GetProcessExecutionDir() + STRING_FLAG(logtail_profile_snapshot);
-    } else {
-        return GetAgentLogDir() + "loongcollector_profile_snapshot";
-    }
-}
-
-
 string GetObserverEbpfHostPath() {
     if (BOOL_FLAG(logtail_mode)) {
         return STRING_FLAG(sls_observer_ebpf_host_path);
@@ -884,14 +866,10 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) {
                        "reader_close_unused_file_time",
                        "ALIYUN_LOGTAIL_READER_CLOSE_UNUSED_FILE_TIME");
 
-    if (confJson.isMember("log_profile_save_interval") && confJson["log_profile_save_interval"].isInt())
-        LogFileProfiler::GetInstance()->SetProfileInterval(confJson["log_profile_save_interval"].asInt());
-
     LOG_DEBUG(sLogger,
               ("logreader delete interval", INT32_FLAG(logreader_filedeleted_remove_interval))(
                   "check handler interval", INT32_FLAG(check_handler_timeout_interval))(
-                  "reader close interval", INT32_FLAG(reader_close_unused_file_time))(
-                  "profile interval", LogFileProfiler::GetInstance()->GetProfileInterval()));
+                  "reader close interval", INT32_FLAG(reader_close_unused_file_time)));
 
 
     if (confJson.isMember("cpu_usage_limit")) {

diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h
@@ -18,12 +18,12 @@
 
 #include <json/json.h>
 
+#include <functional>
 #include <map>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
-#include <functional>
 
 #include "InstanceConfig.h"
 #include "protobuf/sls/sls_logs.pb.h"
@@ -49,9 +49,7 @@ std::string GetInotifyWatcherDirsDumpFileName();
 std::string GetAgentLoggersPrefix();
 std::string GetAgentLogName();
 std::string GetAgentSnapshotDir();
-std::string GetAgentProfileLogName();
 std::string GetAgentStatusLogName();
-std::string GetProfileSnapshotDumpFileName();
 std::string GetObserverEbpfHostPath();
 std::string GetSendBufferFileNamePrefix();
 std::string GetLegacyUserLocalConfigFilePath();
@@ -301,7 +299,7 @@ class AppConfig {
 
 public:
     AppConfig();
-    ~AppConfig(){};
+    ~AppConfig() {};
 
     void LoadInstanceConfig(const std::map<std::string, std::shared_ptr<InstanceConfig>>&);
 

diff --git a/core/application/Application.cpp b/core/application/Application.cpp
@@ -40,7 +40,6 @@
 #include "file_server/event_handler/LogInput.h"
 #include "go_pipeline/LogtailPlugin.h"
 #include "logger/Logger.h"
-#include "monitor/LogFileProfiler.h"
 #include "monitor/MetricExportor.h"
 #include "monitor/Monitor.h"
 #include "pipeline/PipelineManager.h"
@@ -112,7 +111,7 @@ void Application::Init() {
     AppConfig::GetInstance()->LoadAppConfig(GetAgentConfigFile());
 
     // Initialize basic information: IP, hostname, etc.
-    LogFileProfiler::GetInstance();
+    LoongCollectorMonitor::GetInstance();
 #ifdef __ENTERPRISE__
     EnterpriseConfigProvider::GetInstance()->Init("enterprise");
     EnterpriseConfigProvider::GetInstance()->LoadRegionConfig();
@@ -133,25 +132,25 @@ void Application::Init() {
     const string& interface = AppConfig::GetInstance()->GetBindInterface();
     const string& configIP = AppConfig::GetInstance()->GetConfigIP();
     if (!configIP.empty()) {
-        LogFileProfiler::mIpAddr = configIP;
+        LoongCollectorMonitor::mIpAddr = configIP;
         LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_ip", GetHostIp());
     } else if (!interface.empty()) {
-        LogFileProfiler::mIpAddr = GetHostIp(interface);
-        if (LogFileProfiler::mIpAddr.empty()) {
+        LoongCollectorMonitor::mIpAddr = GetHostIp(interface);
+        if (LoongCollectorMonitor::mIpAddr.empty()) {
             LOG_WARNING(sLogger,
                         ("failed to get ip from interface", "try to get any available ip")("interface", interface));
         }
-    } else if (LogFileProfiler::mIpAddr.empty()) {
+    } else if (LoongCollectorMonitor::mIpAddr.empty()) {
         LOG_WARNING(sLogger, ("failed to get ip from hostname or eth0 or bond0", "try to get any available ip"));
     }
-    if (LogFileProfiler::mIpAddr.empty()) {
-        LogFileProfiler::mIpAddr = GetAnyAvailableIP();
-        LOG_INFO(sLogger, ("get available ip succeeded", LogFileProfiler::mIpAddr));
+    if (LoongCollectorMonitor::mIpAddr.empty()) {
+        LoongCollectorMonitor::mIpAddr = GetAnyAvailableIP();
+        LOG_INFO(sLogger, ("get available ip succeeded", LoongCollectorMonitor::mIpAddr));
     }
 
     const string& configHostName = AppConfig::GetInstance()->GetConfigHostName();
     if (!configHostName.empty()) {
-        LogFileProfiler::mHostname = configHostName;
+        LoongCollectorMonitor::mHostname = configHostName;
         LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_hostname", GetHostName());
     }
 
@@ -165,12 +164,12 @@ void Application::Init() {
 #endif
 
     int32_t systemBootTime = AppConfig::GetInstance()->GetSystemBootTime();
-    LogFileProfiler::mSystemBootTime = systemBootTime > 0 ? systemBootTime : GetSystemBootTime();
+    LoongCollectorMonitor::mSystemBootTime = systemBootTime > 0 ? systemBootTime : GetSystemBootTime();
 
     // generate app_info.json
     Json::Value appInfoJson;
-    appInfoJson["ip"] = Json::Value(LogFileProfiler::mIpAddr);
-    appInfoJson["hostname"] = Json::Value(LogFileProfiler::mHostname);
+    appInfoJson["ip"] = Json::Value(LoongCollectorMonitor::mIpAddr);
+    appInfoJson["hostname"] = Json::Value(LoongCollectorMonitor::mHostname);
     appInfoJson["UUID"] = Json::Value(Application::GetInstance()->GetUUID());
     appInfoJson["instance_id"] = Json::Value(Application::GetInstance()->GetInstanceId());
 #ifdef __ENTERPRISE__
@@ -189,16 +188,16 @@ void Application::Init() {
 #define ILOGTAIL_COMPILER VERSION_STR(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
 #endif
     appInfoJson["compiler"] = Json::Value(ILOGTAIL_COMPILER);
-    appInfoJson["os"] = Json::Value(LogFileProfiler::mOsDetail);
+    appInfoJson["os"] = Json::Value(LoongCollectorMonitor::mOsDetail);
     appInfoJson["update_time"] = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S");
     string appInfo = appInfoJson.toStyledString();
     OverwriteFile(GetAgentAppInfoFile(), appInfo);
     LOG_INFO(sLogger, ("app info", appInfo));
 }
 
 void Application::Start() { // GCOVR_EXCL_START
-    LogFileProfiler::mStartTime = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S");
-    LogtailMonitor::GetInstance()->UpdateConstMetric("start_time", LogFileProfiler::mStartTime);
+    LoongCollectorMonitor::mStartTime = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S");
+    LogtailMonitor::GetInstance()->UpdateConstMetric("start_time", LoongCollectorMonitor::mStartTime);
 
 #if defined(__ENTERPRISE__) && defined(_MSC_VER)
     InitWindowsSignalObject();
@@ -287,7 +286,6 @@ void Application::Start() { // GCOVR_EXCL_START
             lastConfigCheckTime = curTime;
         }
         if (curTime - lastProfilingCheckTime >= INT32_FLAG(profiling_check_interval)) {
-            LogFileProfiler::GetInstance()->SendProfileData();
             MetricExportor::GetInstance()->PushMetrics(false);
             lastProfilingCheckTime = curTime;
         }
@@ -330,7 +328,7 @@ void Application::Start() { // GCOVR_EXCL_START
 } // GCOVR_EXCL_STOP
 
 void Application::GenerateInstanceId() {
-    mInstanceId = CalculateRandomUUID() + "_" + LogFileProfiler::mIpAddr + "_" + ToString(mStartTime);
+    mInstanceId = CalculateRandomUUID() + "_" + LoongCollectorMonitor::mIpAddr + "_" + ToString(mStartTime);
 }
 
 bool Application::TryGetUUID() {

diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp
@@ -22,16 +22,16 @@
 
 #include "app_config/AppConfig.h"
 #include "application/Application.h"
-#include "constants/Constants.h"
 #include "common/LogtailCommonFlags.h"
 #include "common/StringTools.h"
 #include "common/UUIDUtil.h"
 #include "common/YamlUtil.h"
 #include "common/version.h"
 #include "config/PipelineConfig.h"
 #include "config/feedbacker/ConfigFeedbackReceiver.h"
+#include "constants/Constants.h"
 #include "logger/Logger.h"
-#include "monitor/LogFileProfiler.h"
+#include "monitor/Monitor.h"
 #include "sdk/Common.h"
 #include "sdk/CurlImp.h"
 #include "sdk/Exception.h"
@@ -206,11 +206,11 @@ string CommonConfigProvider::GetInstanceId() {
 }
 
 void CommonConfigProvider::FillAttributes(configserver::proto::v2::AgentAttributes& attributes) {
-    attributes.set_hostname(LogFileProfiler::mHostname);
-    attributes.set_ip(LogFileProfiler::mIpAddr);
+    attributes.set_hostname(LoongCollectorMonitor::mHostname);
+    attributes.set_ip(LoongCollectorMonitor::mIpAddr);
     attributes.set_version(ILOGTAIL_VERSION);
     google::protobuf::Map<string, string>* extras = attributes.mutable_extras();
-    extras->insert({"osDetail", LogFileProfiler::mOsDetail});
+    extras->insert({"osDetail", LoongCollectorMonitor::mOsDetail});
 }
 
 void addConfigInfoToRequest(const std::pair<const string, logtail::ConfigInfo>& configInfo,

diff --git a/core/config/common_provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp
@@ -26,7 +26,7 @@
 #include "common/StringTools.h"
 #include "common/version.h"
 #include "logger/Logger.h"
-#include "monitor/LogFileProfiler.h"
+#include "monitor/Monitor.h"
 #include "sdk/Common.h"
 #include "sdk/CurlImp.h"
 #include "sdk/Exception.h"
@@ -112,7 +112,8 @@ void LegacyCommonConfigProvider::CheckUpdateThread() {
     }
 }
 
-LegacyCommonConfigProvider::ConfigServerAddress LegacyCommonConfigProvider::GetOneConfigServerAddress(bool changeConfigServer) {
+LegacyCommonConfigProvider::ConfigServerAddress
+LegacyCommonConfigProvider::GetOneConfigServerAddress(bool changeConfigServer) {
     if (0 == mConfigServerAddresses.size()) {
         return ConfigServerAddress("", -1); // No address available
     }
@@ -158,7 +159,7 @@ LegacyCommonConfigProvider::SendHeartbeat(const ConfigServerAddress& configServe
     heartBeatReq.set_agent_id(Application::GetInstance()->GetInstanceId());
     heartBeatReq.set_agent_type("iLogtail");
     attributes.set_version(ILOGTAIL_VERSION);
-    attributes.set_ip(LogFileProfiler::mIpAddr);
+    attributes.set_ip(LoongCollectorMonitor::mIpAddr);
     heartBeatReq.mutable_attributes()->MergeFrom(attributes);
     heartBeatReq.mutable_tags()->MergeFrom({GetConfigServerTags().begin(), GetConfigServerTags().end()});
     heartBeatReq.set_running_status("");
@@ -289,8 +290,9 @@ void LegacyCommonConfigProvider::UpdateRemoteConfig(
     if (ec) {
         StopUsingConfigServer();
         LOG_ERROR(sLogger,
-                  ("failed to create dir for legacy common configs", "stop receiving config from legacy common config server")(
-                      "dir", mPipelineSourceDir.string())("error code", ec.value())("error msg", ec.message()));
+                  ("failed to create dir for legacy common configs",
+                   "stop receiving config from legacy common config server")("dir", mPipelineSourceDir.string())(
+                      "error code", ec.value())("error msg", ec.message()));
         return;
     }
 

diff --git a/core/file_server/ConfigManager.cpp b/core/file_server/ConfigManager.cpp
@@ -36,7 +36,6 @@
 #include "app_config/AppConfig.h"
 #include "checkpoint/CheckPointManager.h"
 #include "common/CompressTools.h"
-#include "constants/Constants.h"
 #include "common/ErrorUtil.h"
 #include "common/ExceptionBase.h"
 #include "common/FileSystemUtil.h"
@@ -46,10 +45,10 @@
 #include "common/StringTools.h"
 #include "common/TimeUtil.h"
 #include "common/version.h"
+#include "constants/Constants.h"
 #include "file_server/EventDispatcher.h"
-#include "file_server/event_handler/EventHandler.h"
 #include "file_server/FileServer.h"
-#include "monitor/LogFileProfiler.h"
+#include "file_server/event_handler/EventHandler.h"
 #include "monitor/AlarmManager.h"
 #include "pipeline/Pipeline.h"
 #include "pipeline/PipelineManager.h"
@@ -101,7 +100,7 @@ DEFINE_FLAG_INT32(docker_config_update_interval, "interval between docker config
 
 namespace logtail {
 
-// 
+//
 ParseConfResult ParseConfig(const std::string& configName, Json::Value& jsonRoot) {
     // Get full path, if it is a relative path, prepend process execution dir.
     std::string fullPath = configName;
@@ -200,7 +199,7 @@ ConfigManager::ConfigManager() {
     // CorrectionLogtailSysConfDir(); // first create dir then rewrite system-uuid file in GetSystemUUID
     // use a thread to get uuid, work around for CalculateDmiUUID hang
     // mUUID = CalculateDmiUUID();
-    // mInstanceId = CalculateRandomUUID() + "_" + LogFileProfiler::mIpAddr + "_" + ToString(time(NULL));
+    // mInstanceId = CalculateRandomUUID() + "_" + LoongCollectorMonitor::mIpAddr + "_" + ToString(time(NULL));
     // ReloadMappingConfig();
 }
 

diff --git a/core/file_server/EventDispatcher.cpp b/core/file_server/EventDispatcher.cpp
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "EventDispatcher.h"
+
 #include "Flags.h"
 #if defined(__linux__)
 #include <fnmatch.h>
@@ -45,7 +46,6 @@
 #include "file_server/event_handler/LogInput.h"
 #include "file_server/polling/PollingDirFile.h"
 #include "file_server/polling/PollingModify.h"
-#include "monitor/LogFileProfiler.h"
 #include "monitor/AlarmManager.h"
 #include "monitor/MetricExportor.h"
 #include "protobuf/sls/metric.pb.h"

diff --git a/core/file_server/EventDispatcher.h b/core/file_server/EventDispatcher.h
@@ -17,24 +17,25 @@
 #pragma once
 #include <sys/types.h>
 #if defined(__linux__)
-#include <sys/socket.h>
-#include <netinet/in.h>
 #include <arpa/inet.h>
 #include <netdb.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
 #include <unistd.h>
 #endif
 #include <stddef.h>
 #include <time.h>
+
+#include <set>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
-#include <set>
-#include "monitor/LogFileProfiler.h"
-#include "file_server/polling/PollingModify.h"
-#include "file_server/polling/PollingDirFile.h"
-#include "file_server/event_listener/EventListener.h"
+
 #include "checkpoint/CheckPointManager.h"
 #include "file_server/FileDiscoveryOptions.h"
+#include "file_server/event_listener/EventListener.h"
+#include "file_server/polling/PollingDirFile.h"
+#include "file_server/polling/PollingModify.h"
 namespace logtail {
 
 class TimeoutHandler;
@@ -184,11 +185,11 @@ class EventDispatcher {
     //  * @return true on success; false on failure
     //  */
     // bool Dispatch();
-// #if defined(_MSC_VER)
-//     virtual void InitWindowsSignalObject() {}
-//     virtual void SyncWindowsSignalObject() {}
-//     virtual void ReleaseWindowsSignalObject() {}
-// #endif
+    // #if defined(_MSC_VER)
+    //     virtual void InitWindowsSignalObject() {}
+    //     virtual void SyncWindowsSignalObject() {}
+    //     virtual void ReleaseWindowsSignalObject() {}
+    // #endif
     // #if defined(__linux__)
     //     virtual void InitShennong() = 0;
     //     virtual void CheckShennong() = 0;

diff --git a/core/file_server/event_handler/EventHandler.cpp b/core/file_server/event_handler/EventHandler.cpp
@@ -1071,18 +1071,6 @@ int32_t ModifyHandler::PushLogToProcessor(LogFileReaderPtr reader, LogBuffer* lo
     int32_t pushRetry = 0;
     if (!logBuffer->rawBuffer.empty()) {
         reader->ReportMetrics(logBuffer->readLength);
-        LogFileProfiler::GetInstance()->AddProfilingReadBytes(reader->GetConfigName(),
-                                                              reader->GetRegion(),
-                                                              reader->GetProject(),
-                                                              reader->GetLogstore(),
-                                                              reader->GetConvertedPath(),
-                                                              reader->GetHostLogPath(),
-                                                              reader->GetExtraTags(),
-                                                              reader->GetDevInode().dev,
-                                                              reader->GetDevInode().inode,
-                                                              reader->GetFileSize(),
-                                                              reader->GetLastFilePos(),
-                                                              time(NULL));
         PipelineEventGroup group = LogFileReader::GenerateEventGroup(reader, logBuffer);
 
         while (!ProcessorRunner::GetInstance()->PushQueue(reader->GetQueueKey(), 0, std::move(group))) // 10ms