Merge branch '2.0' of github.com:alibaba/ilogtail into 2.1

core/application/Application.cpp

@@ -49,6 +49,7 @@
 #include "config/provider/EnterpriseConfigProvider.h"
 #include "config/provider/LegacyConfigProvider.h"
 #if defined(__linux__) && !defined(__ANDROID__)
+#include "common/LinuxDaemonUtil.h"
 #include "shennong/ShennongManager.h"
 #include "streamlog/StreamLogManager.h"
 #endif
@@ -64,6 +65,10 @@ DEFINE_FLAG_INT32(config_scan_interval, "seconds", 10);
 DEFINE_FLAG_INT32(profiling_check_interval, "seconds", 60);
 DEFINE_FLAG_INT32(tcmalloc_release_memory_interval, "force release memory held by tcmalloc, seconds", 300);
 DEFINE_FLAG_INT32(exit_flushout_duration, "exit process flushout duration", 20 * 1000);
+#if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__)
+DEFINE_FLAG_BOOL(enable_cgroup, "", true);
+#endif
+
 
 DECLARE_FLAG_BOOL(send_prefer_real_ip);
 DECLARE_FLAG_BOOL(global_network_success);
@@ -149,6 +154,12 @@ void Application::Init() {
     GenerateInstanceId();
     TryGetUUID();
 
+#if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__)
+    if (BOOL_FLAG(enable_cgroup)) {
+        CreateCGroup();
+    }
+#endif
+
     int32_t systemBootTime = AppConfig::GetInstance()->GetSystemBootTime();
     LogFileProfiler::mSystemBootTime = systemBootTime > 0 ? systemBootTime : GetSystemBootTime();
 

core/common/FileSystemUtil.cpp

@@ -21,7 +21,6 @@
 #include <fnmatch.h>
 #endif
 #include <boost/filesystem.hpp>
-
 #include <fstream>
 
 #include "RuntimeUtil.h"
@@ -164,6 +163,20 @@ bool OverwriteFile(const std::string& fileName, const std::string& content) {
     return true;
 }
 
+bool WriteFile(const std::string& fileName, const std::string& content, std::string& errMsg) {
+    ofstream f(fileName, ios::trunc);
+    if (!f.is_open()) {
+        errMsg = "failed to open file " + fileName;
+        return false;
+    }
+    f.write(content.c_str(), content.size());
+    if (f.fail()) {
+        errMsg = strerror(errno);
+        return false;
+    }
+    return true;
+}
+
 bool IsAccessibleDirectory(const std::string& dirPath) {
     boost::filesystem::directory_iterator end;
     try {

core/common/FileSystemUtil.h

@@ -15,18 +15,19 @@
  */
 
 #pragma once
-#include <string>
-#include <memory>
-#include <vector>
 #include <sys/stat.h>
+
 #include <boost/regex.hpp>
+#include <memory>
+#include <string>
+#include <vector>
 #if defined(__linux__)
 #include <dirent.h>
 #elif defined(_MSC_VER)
 #include <Windows.h>
 #endif
-#include "ErrorUtil.h"
 #include "DevInode.h"
+#include "ErrorUtil.h"
 #include "LogtailCommonFlags.h"
 
 // Filesystem utility.
@@ -89,6 +90,8 @@ bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t
 // OverwriteFile overwrides @fileName with @content.
 bool OverwriteFile(const std::string& fileName, const std::string& content);
 
+bool WriteFile(const std::string& fileName, const std::string& content, std::string& errMsg);
+
 // IsAccessibleDirectory checks if the @dirPath is a existent directory and accessible.
 // Accessibility means that we can iterate the contents of it.
 bool IsAccessibleDirectory(const std::string& dirPath);
@@ -251,9 +254,7 @@ namespace fsutil {
         int64_t GetFileSize() const;
 
         // GetMode returns st_mode.
-        int GetMode() const {
-            return static_cast<int>(mRawStat.st_mode);
-        }
+        int GetMode() const { return static_cast<int>(mRawStat.st_mode); }
     };
 
 } // namespace fsutil

core/go_pipeline/LogtailPlugin.cpp

@@ -50,10 +50,13 @@ LogtailPlugin::LogtailPlugin() {
     mPluginValid = false;
     mPluginAlarmConfig.mLogstore = "logtail_alarm";
     mPluginAlarmConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid);
+    mPluginAlarmConfig.mCompressType = FlusherSLS::CompressType::ZSTD;
     mPluginProfileConfig.mLogstore = "shennong_log_profile";
     mPluginProfileConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid);
+    mPluginProfileConfig.mCompressType = FlusherSLS::CompressType::ZSTD;
     mPluginContainerConfig.mLogstore = "logtail_containers";
     mPluginContainerConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid);
+    mPluginContainerConfig.mCompressType = FlusherSLS::CompressType::ZSTD;
 
     mPluginCfg["LogtailSysConfDir"] = AppConfig::GetInstance()->GetLogtailSysConfDir();
     mPluginCfg["HostIP"] = LogFileProfiler::mIpAddr;

core/monitor/Monitor.cpp

@@ -49,6 +49,7 @@
 #include "config/provider/EnterpriseConfigProvider.h"
 #endif
 #include "pipeline/PipelineManager.h"
+#include "profile_sender/ProfileSender.h"
 
 using namespace std;
 using namespace sls_logs;
@@ -178,11 +179,10 @@ void LogtailMonitor::Monitor() {
                 lastCheckHardLimitTime = monitorTime;
 
                 GetMemStat();
-                CalCpuStat(curCpuStat, mCpuStat);
-                if (CheckHardCpuLimit() || CheckHardMemLimit()) {
+                if (CheckHardMemLimit()) {
                     LOG_ERROR(sLogger,
                               ("Resource used by program exceeds hard limit",
-                               "prepare restart Logtail")("cpu_usage", mCpuStat.mCpuUsage)("mem_rss", mMemStat.mRss));
+                               "prepare restart Logtail")("mem_rss", mMemStat.mRss));
                     Suicide();
                 }
             }
@@ -204,6 +204,7 @@ void LogtailMonitor::Monitor() {
                 // Returning true means too much violations, so we have to prepare to restart
                 // logtail to release resource.
                 // Mainly for controlling memory because we have no idea to descrease memory usage.
+                CalCpuStat(curCpuStat, mCpuStat);
                 if (CheckSoftCpuLimit() || CheckSoftMemLimit()) {
                     LOG_ERROR(sLogger,
                               ("Resource used by program exceeds upper limit for some time",
@@ -471,15 +472,8 @@ bool LogtailMonitor::CheckSoftMemLimit() {
     return false;
 }
 
-bool LogtailMonitor::CheckHardCpuLimit() {
-    float cpuUsageLimit = AppConfig::GetInstance()->IsResourceAutoScale()
-        ? AppConfig::GetInstance()->GetScaledCpuUsageUpLimit()
-        : AppConfig::GetInstance()->GetCpuUsageUpLimit();
-    return mCpuStat.mCpuUsage > 10 * cpuUsageLimit;
-}
-
 bool LogtailMonitor::CheckHardMemLimit() {
-    return mMemStat.mRss > 10 * AppConfig::GetInstance()->GetMemUsageUpLimit();
+    return mMemStat.mRss > 5 * AppConfig::GetInstance()->GetMemUsageUpLimit();
 }
 
 void LogtailMonitor::DumpToLocal(const sls_logs::LogGroup& logGroup) {
@@ -558,6 +552,13 @@ std::string LogtailMonitor::GetLoadAvg() {
     return loadStr;
 }
 
+uint32_t LogtailMonitor::GetCpuCores() {
+    if (!CalCpuCores()) {
+        return 0;
+    }
+    return mCpuCores;
+}
+
 // Get the number of cores in CPU.
 bool LogtailMonitor::CalCpuCores() {
     ifstream fin;

core/monitor/Monitor.h

@@ -24,7 +24,6 @@
 #include "LogtailMetric.h"
 #include "MetricConstants.h"
 #include "MetricStore.h"
-#include "profile_sender/ProfileSender.h"
 #if defined(_MSC_VER)
 #include <Windows.h>
 #endif
@@ -88,6 +87,8 @@ class LogtailMonitor : public MetricStore {
     bool Init();
     void Stop();
 
+    uint32_t GetCpuCores();
+
     // GetRealtimeCpuLevel return a value to indicates current CPU usage level.
     // LogInput use it to do flow control.
     float GetRealtimeCpuLevel() { return mRealtimeCpuStat.mCpuUsage / mScaledCpuUsageUpLimit; }
@@ -114,7 +115,6 @@ class LogtailMonitor : public MetricStore {
     // @return true if the memory usage exceeds limit continuously.
     bool CheckSoftMemLimit();
 
-    bool CheckHardCpuLimit();
     bool CheckHardMemLimit();
 
     // SendStatusProfile collects status profile and send them to server.
@@ -174,7 +174,7 @@ class LogtailMonitor : public MetricStore {
     float mScaledCpuUsageUpLimit;
 #if defined(__linux__)
     const static int32_t CPU_STAT_FOR_SCALE_ARRAY_SIZE = 2;
-    int32_t mCpuCores;
+    int32_t mCpuCores = 0;
     CpuStat mCpuStatForScale;
     OsCpuStat mOsCpuStatForScale;
     // mCpuArrayForScale and mOsCpuArrayForScale store lastest two CPU usage of

core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp

@@ -1577,7 +1577,7 @@ HTTP/2.0' '200' '154' 'go-sdk'"
 
             // run function ProcessorParseDelimiterNative
             ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
-            ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
+            ProcessorInstance processorInstance(&processorParseDelimiterNative, pluginId);
             APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
             processorParseDelimiterNative.Process(eventGroup);